summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPeter Hutterer <peter.hutterer@who-t.net>2013-02-18 16:41:53 +1000
committerPeter Hutterer <peter.hutterer@who-t.net>2013-02-18 16:41:53 +1000
commit8baa66c8c846c70a88ff32a3afa989030413ac51 (patch)
tree08a937f2713a420205ec63251eb70512d0c40430
parentdcc060bdcb228077d015dd16021b85ffb63d024d (diff)
parentc1b48eeb80a00437dca016b980b71052a58286f8 (diff)
Merge branch 'multiple-registries'
Conflicts: registry/server-registry.xml
-rw-r--r--HACKING27
-rw-r--r--Makefile.am6
-rw-r--r--common.mk2
-rw-r--r--registry/server-registry.xml367
-rwxr-xr-xregistry/xit-bug-registry3
-rwxr-xr-xregistry/xit-bug-registry-test97
-rw-r--r--registry/xit.py581
7 files changed, 878 insertions, 205 deletions
diff --git a/HACKING b/HACKING
index 7f1e55a..7fb5ec4 100644
--- a/HACKING
+++ b/HACKING
@@ -222,19 +222,26 @@ track of which tests fail, which is what the bug registry addresses.
Often, what really matters is if there are any tests that changed after a
fix in the server. To use the bug registry for this task run the following
commands. On the __original__ server, run
- # Run the grab tests, printing to a JUnit test xml file
- ./test/server/grab --gtest_output="xml:grab.xml"
- # Create a registry based on the test results
- xit-bug-registry create grab.xml > grab-results.xml
- # fix server bug
- # Re-run grab tests on new server
- ./test/server/grab --gtest_output="xml:grab.xml"
+ # Run all tests. Use -k to ensure all tests are run, even after some have
+ # failed.
+ make -k check
+ # Results end up in $top_builddir/results/latest
+ # Create a registry based on that.
+ xit-bug-registry create results/latest/*.xml > before.xml
+
+ # fix bug
+ # Re-run on new code
+ make -k check
# Compare previous results with new results
- xit-bug-registry verify grab.xml < grab-results.xml
-
+ xit-bug-registry create results/latest/*.xml > after.xml
+ xit-bug-registry compare before.xml after.xml
The output will print the test names and the expected vs real outcome plus a
-status code to grep for the unexpected.
+status code to grep for the unexpected. This will take a while since it runs
+all tests. The shortcut is:
+ # Re-run only the server test
+ ./server --gtest_output="xml:after.xml"
+ xit-bug-registry -f before.xml verify after.xml
Note that especially test failures need to be treated with caution. An
unrelated fix may alter the outcome of a already failing tests (e.g. the
diff --git a/Makefile.am b/Makefile.am
index d8c364d..57f3282 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,10 @@
ACLOCAL_AMFLAGS = -Im4
-SUBDIRS = tests recordings
+SUBDIRS = . tests recordings
EXTRA_DIST = autogen.sh HACKING 99-ignore-virtual-devices.conf 99-tag-virtual-devices.rules
+check-local:
+ @export DATE=`date +%Y-%m-%d-%H:%M`; \
+ mkdir -p $(top_builddir)/results/$$DATE; \
+ ln -sfT $$DATE $(top_builddir)/results/latest
diff --git a/common.mk b/common.mk
index 6d755bd..e3129ed 100644
--- a/common.mk
+++ b/common.mk
@@ -1,5 +1,7 @@
include $(top_srcdir)/Makefile-xorg-gtest.am
+TESTS_ENVIRONMENT=GTEST_OUTPUT="xml:$(top_builddir)/results/latest/"
+
AM_CPPFLAGS = \
$(XI_CFLAGS) \
$(XFIXES_CFLAGS) \
diff --git a/registry/server-registry.xml b/registry/server-registry.xml
index 52dffd6..da7e0a7 100644
--- a/registry/server-registry.xml
+++ b/registry/server-registry.xml
@@ -1,5 +1,5 @@
<xit:registries xmlns:xit="http://www.x.org/xorg-integration-testing">
- <xit:registry name="xorg-upstream">
+ <xit:registry name="server">
<xit:meta>
<xit:date>2013-01-21</xit:date>
<xit:moduleversion name="xserver" type="git" repo="git://git.freedesktop.org/git/xorg/xserver">c1602d1c17967bdd4db9db19b3a9c0dfca6a58aa</xit:moduleversion>
@@ -661,5 +661,370 @@
</xit:testcase>
</xit:testsuite>
</xit:registry>
+ <xit:registry name="input-module-load">
+ <xit:meta>
+ <xit:date>2013-02-05</xit:date>
+ </xit:meta>
+ <xit:testsuite name="InputModuleLoadTest">
+ <xit:testcase name="CheckForLoadFailure/0" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/1" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/10" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/11" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/12" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/13" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/2" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/3" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/4" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/5" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/6" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/7" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/8" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/9" success="true"/>
+ </xit:testsuite>
+ </xit:registry>
+ <xit:registry name="kbd-driver">
+ <xit:meta>
+ <xit:date>2013-02-05</xit:date>
+ </xit:meta>
+ <xit:testsuite name="KeyboardTest">
+ <xit:testcase name="DeviceExists/0" success="true"/>
+ <xit:testcase name="DeviceExists/1" success="true"/>
+ <xit:testcase name="DeviceExists/2" success="true"/>
+ <xit:testcase name="KeyboardLayout/0" success="true"/>
+ <xit:testcase name="KeyboardLayout/1" success="false"/>
+ <xit:testcase name="KeyboardLayout/2" success="false"/>
+ </xit:testsuite>
+ </xit:registry>
+ <xit:registry name="legacy-drivers">
+ <xit:meta>
+ <xit:date>2013-02-05</xit:date>
+ </xit:meta>
+ <xit:testsuite name="AcecadTest">
+ <xit:testcase name="InputDeviceSectionWithOptionDevice" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="AiptekTest">
+ <xit:testcase name="InputDeviceSectionWithType/0" success="true"/>
+ <xit:testcase name="InputDeviceSectionWithType/1" success="true"/>
+ <xit:testcase name="InputDeviceSectionWithType/2" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="ElographicsTest">
+ <xit:testcase name="InputDeviceSectionWithOptionDevice" success="true"/>
+ <xit:testcase name="InvertX" success="true"/>
+ <xit:testcase name="InvertY" success="true"/>
+ <xit:testcase name="StylusMovement" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="LegacyInputDriverTest">
+ <xit:testcase name="InputDeviceSectionSimple/0" success="true"/>
+ <xit:testcase name="InputDeviceSectionSimple/1" success="true"/>
+ <xit:testcase name="InputDeviceSectionSimple/2" success="true"/>
+ <xit:testcase name="InputDeviceSectionSimple/3" success="true"/>
+ <xit:testcase name="InputDeviceSectionSimple/4" success="true"/>
+ <xit:testcase name="InputDeviceSectionSimple/5" success="true"/>
+ <xit:testcase name="InputDeviceSectionSimple/6" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="VoidTest">
+ <xit:testcase name="InputDeviceSectionSimple" success="true"/>
+ </xit:testsuite>
+ </xit:registry>
+ <xit:registry name="evdev-driver">
+ <xit:meta>
+ <xit:date>2013-02-05</xit:date>
+ </xit:meta>
+ <xit:testsuite name="EvdevAxisLabelTest">
+ <xit:testcase name="AbsoluteAxes" success="true"/>
+ <xit:testcase name="RelAndAbsoluteAxes" success="true"/>
+ <xit:testcase name="RelativeAxes" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevButtonMappingTest">
+ <xit:testcase name="ButtonMapping/0" success="true"/>
+ <xit:testcase name="ButtonMapping/1" success="true"/>
+ <xit:testcase name="ButtonMapping/2" success="true"/>
+ <xit:testcase name="ButtonMapping/3" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevFloatingSlaveTest">
+ <xit:testcase name="FloatingDevice" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevInvalidButtonMappingTest">
+ <xit:testcase name="InvalidButtonMapping/0" success="true"/>
+ <xit:testcase name="InvalidButtonMapping/1" success="true"/>
+ <xit:testcase name="InvalidButtonMapping/2" success="true"/>
+ <xit:testcase name="InvalidButtonMapping/3" success="true"/>
+ <xit:testcase name="InvalidButtonMapping/4" success="true"/>
+ <xit:testcase name="InvalidButtonMapping/5" success="true"/>
+ <xit:testcase name="InvalidButtonMapping/6" success="true"/>
+ <xit:testcase name="InvalidButtonMapping/7" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevJoystickTest">
+ <xit:testcase name="MTAxesNoButtons" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevMouseTest">
+ <xit:testcase name="BtnReleaseMaskOnly" success="true"/>
+ <xit:testcase name="DevNode" success="true"/>
+ <xit:testcase name="MiddleButtonEmulation" success="true"/>
+ <xit:testcase name="ScrollWheel" success="true"/>
+ <xit:testcase name="SmoothScrolling" success="true"/>
+ <xit:testcase name="SmoothScrollingAvailable" success="true"/>
+ <xit:testcase name="TerminateWithButtonDown" success="false"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevQEMUTabletTest">
+ <xit:testcase name="AbsoluteAxesWork" success="true"/>
+ <xit:testcase name="HasAbsoluteAxes" success="true"/>
+ <xit:testcase name="HasScrollingAxes" success="true"/>
+ <xit:testcase name="ScrollingWorks" success="true"/>
+ <xit:testcase name="SmoothScrollingWorks" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevTouchRotationTest">
+ <xit:testcase name="AxisSwapInversion/0" success="true"/>
+ <xit:testcase name="AxisSwapInversion/1" success="true"/>
+ <xit:testcase name="AxisSwapInversion/2" success="true"/>
+ <xit:testcase name="AxisSwapInversion/3" success="true"/>
+ <xit:testcase name="AxisSwapInversion/4" success="true"/>
+ <xit:testcase name="AxisSwapInversion/5" success="true"/>
+ <xit:testcase name="AxisSwapInversion/6" success="true"/>
+ <xit:testcase name="AxisSwapInversion/7" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevTouchpadTest">
+ <xit:testcase name="AxisLabels" success="true"/>
+ <xit:testcase name="DeviceExists" success="true"/>
+ <xit:testcase name="PointerMovement" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevXKBConfigRulesTest">
+ <xit:testcase name="NoRuleChangeAllowed" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevXKBConfigTest">
+ <xit:testcase name="XKBOptionParseTest/0" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/1" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/10" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/11" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/12" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/13" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/14" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/15" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/2" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/3" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/4" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/5" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/6" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/7" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/8" success="true"/>
+ <xit:testcase name="XKBOptionParseTest/9" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="EvdevXKBTest">
+ <xit:testcase name="DeviceExists/0" success="true"/>
+ <xit:testcase name="DeviceExists/1" success="true"/>
+ <xit:testcase name="DeviceExists/2" success="true"/>
+ <xit:testcase name="KeyboardLayout/0" success="true"/>
+ <xit:testcase name="KeyboardLayout/1" success="true"/>
+ <xit:testcase name="KeyboardLayout/2" success="true"/>
+ </xit:testsuite>
+ </xit:registry>
+ <xit:registry name="mouse-driver">
+ <xit:meta>
+ <xit:date>2013-02-05</xit:date>
+ </xit:meta>
+ <xit:testsuite name="MouseTest">
+ <xit:testcase name="BtnPress" success="false"/>
+ <xit:testcase name="BtnRelease" success="true"/>
+ <xit:testcase name="DevNode" success="true"/>
+ <xit:testcase name="Move" success="true"/>
+ <xit:testcase name="ScrollWheel" success="true"/>
+ </xit:testsuite>
+ </xit:registry>
+ <xit:registry name="synaptics-driver">
+ <xit:meta>
+ <xit:date>2013-02-05</xit:date>
+ </xit:meta>
+ <xit:testsuite name="SynapticsClickPad">
+ <xit:testcase name="HotPlugSoftButtons" success="false"/>
+ </xit:testsuite>
+ <xit:testsuite name="SynapticsClickpadSoftButtonsRuntimeTest">
+ <xit:testcase name="SoftButtonsFirst" success="true"/>
+ <xit:testcase name="SoftButtonsSecond" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="SynapticsClickpadSoftButtonsTest">
+ <xit:testcase name="LeftClick" success="true"/>
+ <xit:testcase name="LeftClickInDeadArea" success="true"/>
+ <xit:testcase name="RightClick" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="SynapticsClickpadSoftButtonsWithAreaTest">
+ <xit:testcase name="RightClickInDeadArea" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="SynapticsClickpadTest">
+ <xit:testcase name="ClickpadProperties" success="true"/>
+ <xit:testcase name="DisableDevice" success="true"/>
+ <xit:testcase name="DisableDeviceOneFingerDown" success="true"/>
+ <xit:testcase name="DisableDeviceOneFingerDownAndLift" success="true"/>
+ <xit:testcase name="DisableDeviceOneFingerResume" success="true"/>
+ <xit:testcase name="DisableDeviceOneFingerTwoFingersResume" success="true"/>
+ <xit:testcase name="DisableDeviceTwoFingersDown" success="true"/>
+ <xit:testcase name="DisableDeviceTwoFingersDownAndLift" success="true"/>
+ <xit:testcase name="DisableDeviceTwoFingersOneFingerResume" success="true"/>
+ <xit:testcase name="DisableDeviceTwoFingersResume" success="true"/>
+ <xit:testcase name="Tap" success="true"/>
+ <xit:testcase name="VertScrollDown" success="true"/>
+ <xit:testcase name="VertScrollUp" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="SynapticsSmoothScrollTest">
+ <xit:testcase name="ScrollDelta/0" success="true"/>
+ <xit:testcase name="ScrollDelta/1" success="true"/>
+ <xit:testcase name="ScrollDelta/2" success="true"/>
+ <xit:testcase name="ScrollDelta/3" success="true"/>
+ <xit:testcase name="ScrollDelta/4" success="true"/>
+ <xit:testcase name="ScrollDelta/5" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="SynapticsTest">
+ <xit:testcase name="DevicePresent" success="true"/>
+ <xit:testcase name="ScrollWheel" success="true"/>
+ <xit:testcase name="SmoothScrollingAvailable" success="true"/>
+ <xit:testcase name="TapAndDragEvent" success="true"/>
+ <xit:testcase name="TapEvent" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="SynapticsWarpTest">
+ <xit:testcase name="WarpScaling/0" success="true"/>
+ <xit:testcase name="WarpScaling/1" success="true"/>
+ <xit:testcase name="WarpScaling/2" success="true"/>
+ <xit:testcase name="WarpScaling/3" success="true"/>
+ <xit:testcase name="WarpScaling/4" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="SynapticsWarpXineramaTest">
+ <xit:testcase name="WarpScaling/0" success="true"/>
+ <xit:testcase name="WarpScaling/1" success="true"/>
+ <xit:testcase name="WarpScaling/2" success="true"/>
+ <xit:testcase name="WarpScaling/3" success="true"/>
+ <xit:testcase name="WarpScaling/4" success="true"/>
+ <xit:testcase name="WarpScalingRelative" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="SynapticsWarpZaphodTest">
+ <xit:testcase name="WarpScaling/0" success="true"/>
+ <xit:testcase name="WarpScaling/1" success="true"/>
+ <xit:testcase name="WarpScaling/2" success="true"/>
+ <xit:testcase name="WarpScaling/3" success="true"/>
+ <xit:testcase name="WarpScaling/4" success="true"/>
+ </xit:testsuite>
+ </xit:registry>
+ <xit:registry name="wacom-driver">
+ <xit:meta>
+ <xit:date>2013-02-05</xit:date>
+ </xit:meta>
+ <xit:testsuite name="WacomDriver">
+ <xit:testcase name="PrivToolDoubleFree" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="WacomDriverTest">
+ <xit:testcase name="DeviceNames/0" success="true"/>
+ <xit:testcase name="DeviceNames/1" success="true"/>
+ <xit:testcase name="DeviceNames/10" success="true"/>
+ <xit:testcase name="DeviceNames/11" success="true"/>
+ <xit:testcase name="DeviceNames/12" success="false"/>
+ <xit:testcase name="DeviceNames/13" success="false"/>
+ <xit:testcase name="DeviceNames/14" success="true"/>
+ <xit:testcase name="DeviceNames/2" success="true"/>
+ <xit:testcase name="DeviceNames/3" success="true"/>
+ <xit:testcase name="DeviceNames/4" success="true"/>
+ <xit:testcase name="DeviceNames/5" success="true"/>
+ <xit:testcase name="DeviceNames/6" success="true"/>
+ <xit:testcase name="DeviceNames/7" success="true"/>
+ <xit:testcase name="DeviceNames/8" success="true"/>
+ <xit:testcase name="DeviceNames/9" success="false"/>
+ <xit:testcase name="DeviceType/0" success="true"/>
+ <xit:testcase name="DeviceType/1" success="true"/>
+ <xit:testcase name="DeviceType/10" success="true"/>
+ <xit:testcase name="DeviceType/11" success="true"/>
+ <xit:testcase name="DeviceType/12" success="false"/>
+ <xit:testcase name="DeviceType/13" success="true"/>
+ <xit:testcase name="DeviceType/14" success="true"/>
+ <xit:testcase name="DeviceType/2" success="true"/>
+ <xit:testcase name="DeviceType/3" success="true"/>
+ <xit:testcase name="DeviceType/4" success="true"/>
+ <xit:testcase name="DeviceType/5" success="true"/>
+ <xit:testcase name="DeviceType/6" success="true"/>
+ <xit:testcase name="DeviceType/7" success="true"/>
+ <xit:testcase name="DeviceType/8" success="true"/>
+ <xit:testcase name="DeviceType/9" success="true"/>
+ <xit:testcase name="Rotation/0" success="true"/>
+ <xit:testcase name="Rotation/1" success="true"/>
+ <xit:testcase name="Rotation/10" success="true"/>
+ <xit:testcase name="Rotation/11" success="true"/>
+ <xit:testcase name="Rotation/12" success="false"/>
+ <xit:testcase name="Rotation/13" success="false"/>
+ <xit:testcase name="Rotation/14" success="true"/>
+ <xit:testcase name="Rotation/2" success="true"/>
+ <xit:testcase name="Rotation/3" success="true"/>
+ <xit:testcase name="Rotation/4" success="true"/>
+ <xit:testcase name="Rotation/5" success="true"/>
+ <xit:testcase name="Rotation/6" success="true"/>
+ <xit:testcase name="Rotation/7" success="true"/>
+ <xit:testcase name="Rotation/8" success="true"/>
+ <xit:testcase name="Rotation/9" success="false"/>
+ </xit:testsuite>
+ <xit:testsuite name="WacomHoveringTest">
+ <xit:testcase name="HoveringTest" success="false"/>
+ </xit:testsuite>
+ <xit:testsuite name="WacomLensCursorTest">
+ <xit:testcase name="CursorMove/0" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="WacomMatrixTest">
+ <xit:testcase name="DevicePresent" success="true"/>
+ <xit:testcase name="InputMatrix" success="false"/>
+ </xit:testsuite>
+ <xit:testsuite name="WacomPropertyTest">
+ <xit:testcase name="Button1DoubleMiddleClick" success="false"/>
+ <xit:testcase name="ButtonActionHighButtonValue" success="false"/>
+ <xit:testcase name="ButtonActionInvalidFormat" success="false"/>
+ <xit:testcase name="ButtonActionInvalidType" success="false"/>
+ <xit:testcase name="ButtonActionKeyPress" success="false"/>
+ <xit:testcase name="ButtonActionKeyPressRelease" success="false"/>
+ <xit:testcase name="ButtonActionPropertiesPresent" success="false"/>
+ <xit:testcase name="ButtonActionPropertySetToNone" success="false"/>
+ <xit:testcase name="ButtonActionPropertyUnset" success="false"/>
+ </xit:testsuite>
+ <xit:testsuite name="WacomToolProximityTest">
+ <xit:testcase name="ToolMovesOnProximity/0" success="true"/>
+ <xit:testcase name="ToolMovesOnProximity/1" success="true"/>
+ <xit:testcase name="ToolMovesOnProximity/2" success="true"/>
+ <xit:testcase name="ToolMovesOnProximity/3" success="true"/>
+ <xit:testcase name="ToolMovesOnProximity/4" success="true"/>
+ <xit:testcase name="ToolMovesOnProximity/5" success="true"/>
+ <xit:testcase name="ToolMovesOnProximity/6" success="true"/>
+ </xit:testsuite>
+ </xit:registry>
+ <xit:registry name="libXi">
+ <xit:meta>
+ <xit:date>2013-02-05</xit:date>
+ </xit:meta>
+ <xit:testsuite name="libXiTest">
+ <xit:testcase name="DisplayNotGarbage" success="true"/>
+ </xit:testsuite>
+ <xit:testsuite name="libXiTouchTest">
+ <xit:testcase name="CopyRawTouchEvent" success="true"/>
+ </xit:testsuite>
+ </xit:registry>
+ <xit:registry name="video-module-load">
+ <xit:meta>
+ <xit:date>2013-02-05</xit:date>
+ </xit:meta>
+ <xit:testsuite name="VideoModuleLoadTest">
+ <xit:testcase name="CheckForLoadFailure/0" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/1" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/10" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/11" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/12" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/13" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/14" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/15" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/16" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/17" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/18" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/19" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/2" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/20" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/3" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/4" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/5" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/6" success="true"/>
+ <xit:testcase name="CheckForLoadFailure/7" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/8" success="false"/>
+ <xit:testcase name="CheckForLoadFailure/9" success="false"/>
+ </xit:testsuite>
+ </xit:registry>
</xit:registries>
diff --git a/registry/xit-bug-registry b/registry/xit-bug-registry
index 1eddc0e..8e90fd6 100755
--- a/registry/xit-bug-registry
+++ b/registry/xit-bug-registry
@@ -38,6 +38,9 @@
#
import xit
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
cli = xit.XITTestRegistryCLI()
cli.run()
diff --git a/registry/xit-bug-registry-test b/registry/xit-bug-registry-test
index 0d985f1..3087189 100755
--- a/registry/xit-bug-registry-test
+++ b/registry/xit-bug-registry-test
@@ -9,6 +9,9 @@ import time
import os
import shutil
+REGISTRY_SOURCE_FILE = "server-registry.xml"
+REGISTRY_FILENAME = "/tmp/tmp.registry.xml"
+
class TestXITBug(unittest.TestCase):
def test_defaults(self):
@@ -265,6 +268,20 @@ class TestXITTestRegistry(unittest.TestCase):
return reg2
+ def to_from_xml_multiple(self, reg):
+ xml = reg.toXML()
+
+ f = tempfile.TemporaryFile()
+ f.write(xml)
+ f.seek(0)
+
+ reg2 = xit.XITTestRegistry.fromXML(f)
+ xml2 = reg2[0].toXML(reg2)
+
+ self.assertEqual(xml, xml2)
+
+ return reg2
+
def test_to_from_xml(self):
reg = xit.XITTestRegistry()
self.to_from_xml(reg);
@@ -288,27 +305,47 @@ class TestXITTestRegistry(unittest.TestCase):
for mod, mod2 in zip(reg.moduleversions, reg2.moduleversions):
self.assertEqual(mod, mod2)
+ def test_multiple_registries(self):
+ regs = xit.XITTestRegistry.fromXML(REGISTRY_SOURCE_FILE)
+ self.assertGreater(len(regs), 1)
+ for r in regs:
+ self.assertGreater(len(r.name), 0)
+ self.assertGreater(len(r.tests), 0)
+
+ def test_multiple_registries_write_load(self):
+ regs = xit.XITTestRegistry.fromXML(REGISTRY_SOURCE_FILE)
+ xml = regs[0].toXML(regs)
+
+ f = tempfile.TemporaryFile()
+ f.write(xml)
+ f.seek(0)
+
+ regs2 = xit.XITTestRegistry.fromXML(f)
+ self.assertEqual(len(regs), len(regs2))
+ xml2 = regs2[0].toXML(regs2)
+
+ self.assertEqual(xml, xml2)
class TestXITTestRegistryCLI(unittest.TestCase):
+ EXIT_GENERAL_FAILURE = 1
EXIT_TOO_FEW_ARGS_ERROR_CODE = 2
- REGISTRY_SOURCE_FILE = "server-registry.xml"
- REGISTRY_FILENAME = "/tmp/tmp.registry.xml"
- RESULTS_FILE = "../server-results.xml"
+ RESULTS_FILE = "../server.xml"
def setUp(self):
"""Make sure server-registry.xml is present, which it should be in
this directory"""
- self.assertTrue(os.path.exists(self.REGISTRY_SOURCE_FILE))
+ self.assertTrue(os.path.exists(REGISTRY_SOURCE_FILE))
self.assertTrue(os.path.exists(self.RESULTS_FILE))
- shutil.copyfile(self.REGISTRY_SOURCE_FILE, self.REGISTRY_FILENAME)
+ shutil.copyfile(REGISTRY_SOURCE_FILE, REGISTRY_FILENAME)
self.cli = xit.XITTestRegistryCLI()
- self.registry = self.cli.load_registry_from_file(self.REGISTRY_FILENAME)
+ # FIXME: for now only work on the first one
+ self.registry = self.cli.load_registry_from_file(REGISTRY_FILENAME)[0]
# default args
- self.args = ["-f", self.REGISTRY_FILENAME]
+ self.args = ["-f", REGISTRY_FILENAME]
def test_load_registry(self):
noargs = ""
@@ -324,10 +361,24 @@ class TestXITTestRegistryCLI(unittest.TestCase):
self.cli.run(self.args)
self.assertEquals(e.exception.code, self.EXIT_TOO_FEW_ARGS_ERROR_CODE)
+ # with a registry that doesn't exist
+ with self.assertRaises(SystemExit) as e:
+ self.cli.run(self.args + ["-r", "doesnotexist"])
+ self.assertEquals(e.exception.code, self.EXIT_TOO_FEW_ARGS_ERROR_CODE);
+
+ # with a registry that does exist
+ with self.assertRaises(SystemExit) as e:
+ self.cli.run(self.args + ["-r", "server"])
+ self.assertEquals(e.exception.code, self.EXIT_TOO_FEW_ARGS_ERROR_CODE);
+
def test_list_registry(self):
- args = self.args
- args.append("list")
- self.cli.run(args)
+ self.cli.run(self.args + ["list"])
+
+ with self.assertRaises(SystemExit) as e:
+ self.cli.run(self.args + ["-r", "doesnotexist", "list"])
+ self.assertEquals(e.exception.code, self.EXIT_GENERAL_FAILURE);
+
+ self.cli.run(self.args + ["-r", "server", "list"])
def test_show_info(self):
args = self.args
@@ -345,14 +396,17 @@ class TestXITTestRegistryCLI(unittest.TestCase):
self.cli.run(args + [suite] + [test])
def test_compare(self):
- args = self.args
- args += ["compare", self.REGISTRY_FILENAME, self.REGISTRY_FILENAME]
- self.cli.run(args)
+ self.cli.run(self.args + ["compare", REGISTRY_FILENAME, REGISTRY_FILENAME])
+ self.cli.run(self.args + ["-r", "server", "compare", REGISTRY_FILENAME, REGISTRY_FILENAME])
+
+ with self.assertRaises(SystemExit) as e:
+ self.cli.run(self.args + ["-r", "doesnotexist", "compare", REGISTRY_FILENAME, REGISTRY_FILENAME])
+ self.assertEquals(e.exception.code, self.EXIT_GENERAL_FAILURE);
def test_merge(self):
- args = self.args
- args += ["merge", self.REGISTRY_FILENAME, self.REGISTRY_FILENAME]
- self.cli.run(args)
+
+ self.cli.run(self.args + ["-r", "server", "merge", REGISTRY_FILENAME, REGISTRY_FILENAME])
+ self.cli.run(self.args + ["merge", REGISTRY_FILENAME, REGISTRY_FILENAME])
def test_edit(self):
args = self.args
@@ -398,12 +452,13 @@ class TestXITTestRegistryCLI(unittest.TestCase):
self.cli.run(args + ["--name", "somename", self.RESULTS_FILE])
self.cli.run(args + ["--auto-modversion", "rpm", self.RESULTS_FILE])
- def test_create(self):
- args = self.args
- args.append("verify")
+ def test_verify(self):
+ self.cli.run(self.args + ["verify", self.RESULTS_FILE])
+ self.cli.run(self.args + ["verify", "--check-all", self.RESULTS_FILE])
- self.cli.run(args + [self.RESULTS_FILE])
- self.cli.run(args + ["--check-all", self.RESULTS_FILE])
+ with self.assertRaises(SystemExit) as e:
+ self.cli.run(self.args + ["-r", "evdev", "verify", "--check-all", self.RESULTS_FILE])
+ self.assertEquals(e.exception.code, self.EXIT_GENERAL_FAILURE);
if __name__ == '__main__':
unittest.main()
diff --git a/registry/xit.py b/registry/xit.py
index cd0d49a..b37465b 100644
--- a/registry/xit.py
+++ b/registry/xit.py
@@ -45,6 +45,7 @@ from lxml import objectify
import lxml.etree
import shutil
import subprocess
+import logging
DEFAULT_MODULES = ["xorg-x11-server-Xorg",
"xorg-x11-drv-evdev",
@@ -53,8 +54,11 @@ DEFAULT_MODULES = ["xorg-x11-server-Xorg",
"xorg-x11-drv-mouse",
"xorg-x11-drv-keyboard"]
-def debug(msg):
- print >> sys.stderr, msg
+class Colors:
+ DEFAULT = 0
+ RED = 1
+ GREEN = 2
+ BLUE = 3
XMLNS = "http://www.x.org/xorg-integration-testing"
def xmlns_tag(tag, ns=XMLNS):
@@ -79,6 +83,14 @@ class termcolors:
BLUE = '\033[1;34m'
@classmethod
+ def color(self, color):
+ colors = { Colors.DEFAULT : termcolors.DEFAULT,
+ Colors.RED : termcolors.RED,
+ Colors.GREEN : termcolors.GREEN,
+ Colors.BLUE : termcolors.BLUE }
+ return colors[color]
+
+ @classmethod
def disable(self):
self.DEFAULT = ''
self.RED = ''
@@ -88,12 +100,13 @@ class termcolors:
class XITTestRegistry:
"""Central class keeping a set of test cases and their results"""
- def __init__(self, name="", test_cases = []):
+ def __init__(self, name="", test_cases = [], path="stdin"):
"""Initialise with a registry name and a list of """
self.tests = self._from_list(test_cases)
self.name = name
self.date = time.localtime()
self.moduleversions = []
+ self.path = path
@classmethod
def fromXML(self, filename):
@@ -101,7 +114,7 @@ class XITTestRegistry:
registries = objectify.parse(filename).getroot()
regs = []
for registry in registries.iterchildren(tag=xmlns_tag("registry")):
- reg = XITTestRegistry(name=registry.attrib["name"])
+ reg = XITTestRegistry(name=registry.attrib["name"], path=filename)
for meta in registry.iterchildren(tag=xmlns_tag("meta")):
date = meta.find(xmlns_tag("date"))
@@ -113,7 +126,11 @@ class XITTestRegistry:
type = modversion.attrib["type"]
except KeyError:
type = "git"
- mv = XITModuleVersion(modversion.attrib["name"], modversion.text, type)
+ try:
+ repo = modversion.attrib["repo"]
+ except KeyError:
+ repo = None
+ mv = XITModuleVersion(modversion.attrib["name"], modversion.text, type, repo)
reg.moduleversions.append(mv)
for suite in registry.iterchildren(tag=xmlns_tag("testsuite")):
@@ -143,56 +160,67 @@ class XITTestRegistry:
regs.append(reg)
return regs
- def toXML(self):
- """Generate XML output from this registry and return it"""
+ def toXML(self, others=[]):
+ """Generate XML output from this registry and return it. If others
+ is a list, the resulting XML file contains all of the registries
+ provided including self. Otherwise, only self is written out."""
NSMAP = { "xit" : "http://www.x.org/xorg-integration-testing" }
E = objectify.ElementMaker(annotate = False,
namespace = NSMAP['xit'],
nsmap = NSMAP)
- xit_registries = E.registries()
- xit_registry = E.registry()
- xit_registry.set("name", self.name)
- xit_registries.append(xit_registry)
-
- xit_meta = E.meta()
- xit_registry.append(xit_meta)
- xit_date = E.date(time.strftime("%Y-%m-%d", self.date))
- xit_meta.append(xit_date)
-
- for modversion in sorted(self.moduleversions):
- xit_modversion = E.moduleversion(modversion.version)
- xit_modversion.set("name", modversion.module)
- xit_modversion.set("type", modversion.type)
- xit_meta.append(xit_modversion)
+ xit_registries = E.registries()
- for suite_name, suite in sorted(self.tests.iteritems()):
- xit_suite = E.testsuite()
- xit_suite.set("name", suite_name)
- for name, test in sorted(suite.iteritems()):
- xit_testcase = E.testcase()
- xit_testcase.set("name", test.name)
- xit_testcase.set("success", str(test.status).lower())
-
- for bug in test.getBugs():
- xit_bug = E.bug(bug.url)
- xit_bug.set("type", bug.type)
- xit_testcase.append(xit_bug)
-
- for fix in test.getFixes():
- xit_fix = E.fix(fix.text)
- xit_fix.set("type", fix.type);
- for arg, value in fix.extra_args.iteritems():
- xit_fix.set(arg, value)
- xit_testcase.append(xit_fix)
-
- for info in test.getInfo():
- xit_info = E.testinfo(info.text)
- xit_info.set("type", info.type)
- xit_testcase.append(xit_info)
-
- xit_suite.append(xit_testcase)
- xit_registry.append(xit_suite)
+ if len(others) == 0:
+ others = [self] # yikes
+
+ for r in others:
+ xit_registry = E.registry()
+ xit_registry.set("name", r.name)
+ xit_registries.append(xit_registry)
+
+ xit_meta = E.meta()
+ xit_registry.append(xit_meta)
+
+ xit_date = E.date(time.strftime("%Y-%m-%d", r.date))
+ xit_meta.append(xit_date)
+
+ for modversion in sorted(r.moduleversions):
+ xit_modversion = E.moduleversion(modversion.version)
+ xit_modversion.set("name", modversion.module)
+ xit_modversion.set("type", modversion.type)
+ if modversion.repo:
+ xit_modversion.set("repo", modversion.repo)
+
+ xit_meta.append(xit_modversion)
+
+ for suite_name, suite in sorted(r.tests.iteritems()):
+ xit_suite = E.testsuite()
+ xit_suite.set("name", suite_name)
+ for name, test in sorted(suite.iteritems()):
+ xit_testcase = E.testcase()
+ xit_testcase.set("name", test.name)
+ xit_testcase.set("success", str(test.status).lower())
+
+ for bug in test.getBugs():
+ xit_bug = E.bug(bug.url)
+ xit_bug.set("type", bug.type)
+ xit_testcase.append(xit_bug)
+
+ for fix in test.getFixes():
+ xit_fix = E.fix(fix.text)
+ xit_fix.set("type", fix.type);
+ for arg, value in fix.extra_args.iteritems():
+ xit_fix.set(arg, value)
+ xit_testcase.append(xit_fix)
+
+ for info in test.getInfo():
+ xit_info = E.testinfo(info.text)
+ xit_info.set("type", info.type)
+ xit_testcase.append(xit_info)
+
+ xit_suite.append(xit_testcase)
+ xit_registry.append(xit_suite)
lxml.etree.cleanup_namespaces(xit_registries)
return lxml.etree.tostring(xit_registries, pretty_print=True)
@@ -227,6 +255,8 @@ class XITTestRegistry:
self.tests[test.suite] = {}
self.tests[test.suite][test.name] = test
+ def __cmp__(self, other):
+ return cmp(self.name, other)
class XITTestCase:
"""Represents one single test case, comprised of test suite name and test case name.
@@ -462,9 +492,10 @@ class XITInfoURL(XITInfo):
class XITModuleVersion:
"""Represents a module version of a particular component."""
- def __init__(self, module, version, type = "git"):
+ def __init__(self, module, version, type = "git", repo=None):
self.module = module
self.version = version
+ self.repo = repo
self.type = type
def __str__(self):
@@ -548,46 +579,169 @@ class JUnitTestFailure:
return self.message
+class XITCLIPrinter:
+ def __init__(self):
+ pass
+
+ def separator(self, text):
+ print ":" * 20 + " {:<30} ".format(text) + ":" * 58
+
+ def print_values(self, headers, values, colors = None, section=None):
+ if len(headers) != len(values[0]):
+ logging.error("Mismatched header/values tuples. Refusing to print")
+ return
+
+ if section != None:
+ self.separator(section)
+
+ cw = [] # column width
+ separators = []
+ for header in headers:
+ cw.append(len(header))
+ separators.append("-" * len(header))
+
+ for val in values:
+ for i in range(len(val)):
+ cw[i] = max(cw[i], len(str(val[i])))
+
+ format_str = ""
+ for idx, w in zip(range(len(cw)), cw):
+ format_str += "{%d:<%d}" % (idx, w + 1)
+
+ print format_str.format(*headers)
+ print format_str.format(*separators)
+
+ if not colors:
+ colors = [Colors.DEFAULT] * len(values)
+
+ for color, val in zip(colors, values):
+ print termcolors.color(color) + format_str.format(*val) + termcolors.DEFAULT
+
+
+class XITPrinterPicker(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string=None):
+ if values == "html":
+ printer = XITHTMLPrinter()
+ else:
+ printer = XITCLIPrinter()
+
+ setattr(namespace, "printer", printer)
+
+
+class XITHTMLPrinter:
+ def __init__(self):
+ self.print_html_header()
+
+ def __del__(self):
+ self.print_html_footer()
+
+ def print_html_header(self):
+ print """<html> <head>
+ <style>
+ body{font-family:sans-serif;}
+ .values {border-collapse:collapse;font-size:80%;}
+ .values td, th {padding:0.5em 1em; border:1px solid #ccc; text-align:left;} values {background-color:#eee;text-align:left; font-size:80%;}
+ .red { background-color:#ee0000; text: #000000; }
+ </style>
+ </head><body>
+ """
+
+ def print_html_footer(self):
+ print """</body></html>
+ """
+
+ def print_table_header(self):
+ print """<table class=\"values\">"""
+
+ def print_table_footer(self):
+ print """</table>"""
+
+ def print_values(self, headers, values, colors = None, section = None):
+ if section:
+ print "<h2>%s</h2>" % section
+ self.print_table_header()
+
+ format_str = "<tr>"
+ format_str += "<th>{}</th>" * len(headers)
+ format_str += "</tr>"
+ print format_str.format(*headers)
+
+ if not colors:
+ colors = [Colors.DEFAULT] * len(values)
+
+ for color, val in zip(colors, values):
+ if color == Colors.RED:
+ format_str = "<tr class=\"red\">"
+ else:
+ format_str = "<tr>"
+
+ format_str += "<td>{}</td>" * len(val)
+ format_str += "</tr>"
+ print format_str.format(*val)
+
+ self.print_table_footer()
+
class XITTestRegistryCLI:
+ """Command-line interface to the registry"""
+
def list_tests(self, args):
"""List all tests, showing test name and expected status"""
- registry = self.load_registry(args)
- all_tests = registry.listTestNames()
- all_tests.insert(0, ("TestSuite", "TestCase", "Success"))
- all_tests.insert(1, ("---------", "--------", "-------"))
- for suite, test, status in all_tests:
- print "{:<50}{:<50}{:>10}".format(suite, test, str(status))
+ registries = self.load_registries(args)
+ for r in registries:
+ all_tests = r.listTestNames()
+
+ headers = ("TestSuite", "TestCase", "Success")
+ values = [(suite, test, str(status)) for suite, test, status in all_tests]
+ colors = [Colors.RED if not status else Colors.DEFAULT for (suite, test, status) in all_tests]
+
+ args.printer.print_values(headers, values, colors = colors, section = r.name)
def show_test_info(self, args):
"""Show all information about a given XIT registry test case"""
- registry = self.load_registry(args)
- test = registry.getTest(args.testsuite[0], args.testcase[0])
+ registry = self.load_one_registry(args)
+ test = registry.getTest(args.testsuite, args.testcase)
if test != None:
print str(test)
+ else:
+ logging.error("Unable to find test %s.%s\n", args.testsuite, args.testcase)
- def verify_one_result(self, test, result, format_str):
+ def verify_one_result(self, test, result):
"""Verify the test result given against the registry. Prints a status
message comprising full test name, expected outcome, actual outcome
and a grep-able bit to indicate which way the outcome differs"""
- color = termcolors.DEFAULT
- if test != None:
+ color = Colors.DEFAULT
+
+ if test != None and result != None:
+ suite = test.suite
+ name = test.name
+ status = str(result.status).lower()
expected_status = str(test.status).lower()
if str(test.status).lower() != str(result.status).lower():
status_match = "XX"
if result.status == True:
- color = termcolors.GREEN
+ color = Colors.GREEN
else:
- color = termcolors.RED
+ color = Colors.RED
else:
status_match = "++" if test.status else "--"
+ elif test != None and result == None:
+ suite = test.suite
+ name = test.name
+ status_match = "??"
+ expected_status = str(test.status).lower()
+ status = ""
+ color = Colors.BLUE
else:
+ suite = result.suite
+ name = result.name
+ status = str(result.status).lower()
expected_status = ""
status_match = "??"
- color = termcolors.BLUE
+ color = Colors.BLUE
- print color + format_str.format(status_match, result.suite, result.name, str(result.status).lower(), expected_status) + termcolors.DEFAULT
+ return (color, (status_match, suite, name, status, expected_status))
def check_installed_rpms(self, registry):
@@ -601,26 +755,38 @@ class XITTestRegistryCLI:
def verify_results(self, args):
- """Verify a JUnit test result against the XIT test registry"""
- registry = self.load_registry(args)
- results = JUnitTestResult.fromXML(args.results)
-
- sname_len = 0
- tname_len = 0
+ """Verify a JUnit test result against the XIT test registry.
+ If a registry name is given, compare to that. If none is given
+ and there is more than one registry, search for one named the
+ same as the results file. Otherwise, bail out."""
+ registry = None
+ registries = self.load_registries(args)
+ if not args.regname:
+ regname = os.path.basename(args.results).split(".xml")[0]
+ for r in registries:
+ if r.name == regname:
+ registry = r
+ break
+ if registry == None:
+ logging.error("Failed to match %s with a registry." % args.results)
+ sys.exit(1)
+ else:
+ registry = registries[0]
- sname_len = max([ len(x.suite) for x in results ]) + 1
- tname_len = max([ len(x.name) for x in results ]) + 1
+ results = JUnitTestResult.fromXML(args.results)
if args.check_all:
self.check_installed_rpms(registry)
- format_str = "{0:<5}{1:<%d}{2:<%d}{3:>10}{4:>10}" % (sname_len, tname_len)
- print format_str.format("Code", "TestSuite", "TestCase", "Result", "Expected")
- print format_str.format("----", "---------", "--------", "------", "--------")
-
+ headers = ("Code", "TestSuite", "TestCase", "Result", "Expected")
+ values = []
+ colors = []
for result in sorted(results):
- self.verify_one_result(registry.getTest(result.suite, result.name), result, format_str)
+ c, v = self.verify_one_result(registry.getTest(result.suite, result.name), result)
+ colors.append(c)
+ values.append(v)
+ args.printer.print_values(headers, values, colors = colors, section = registry.name)
def print_modversions(self, name, v1, v2, use_colors=True):
if use_colors and v1 != v2:
@@ -630,71 +796,90 @@ class XITTestRegistryCLI:
format_str = "{0:<30} {1:<50} {2:<50}"
print color + format_str.format(name, v1, v2) + termcolors.DEFAULT
- def compare_meta(self, reg1, reg2):
- self.print_modversions("Module name", reg1.name, reg2.name, False)
- self.print_modversions("-----------", "----------", "----------", False)
-
+ def get_meta_comparison(self, reg1, reg2):
+ modules = [ r.module for r in reg1.moduleversions]
+ modules.extend(r.module for r in reg2.moduleversions if r.name not in modules)
- r1_iter = iter(sorted(reg1.moduleversions))
- r2_iter = iter(sorted(reg2.moduleversions))
-
- try:
- r1_modversion = r1_iter.next()
- r2_modversion = r2_iter.next()
- while True:
- rc = cmp(r1_modversion, r2_modversion)
- if rc == 0:
- self.print_modversions(r1_modversion.module, r1_modversion.version, r2_modversion.version)
- r1_modversion = r1_iter.next()
- r2_modversion = r2_iter.next()
- elif rc > 0:
- self.print_modversions(r2_modversion.module, "???", r2_modversion.version)
- r2_modversion = r2_iter.next()
- elif rc < 0:
- self.print_modversions(r1_modversion.module, r1_modversion.version, "???")
- r1_modversion = r1_iter.next()
- except StopIteration:
- pass
-
- try:
- while True:
- r1_modversion = r1_iter.next()
- self.print_modversions(r1_modversion.module, r1_modversion.version, "???")
- except StopIteration:
- pass
- try:
- while True:
- r2_modversion = r2_iter.next()
- self.print_modversions(r2_modversion.module, "???", r2_modversion.version)
- except StopIteration:
- pass
+ ret = []
+ for m in modules:
+ r1 = "".join([ mr1.version for mr1 in reg1.moduleversions if mr1.module == m ])
+ r2 = "".join([ mr2.version for mr2 in reg2.moduleversions if mr2.module == m ])
+ ret.append((m, r1, r2))
- print
+ return ret
def compare_registries(self, args):
- reg1 = XITTestRegistry.fromXML(args.reg1[0])[0]
- reg2 = XITTestRegistry.fromXML(args.reg2[0])[0]
+ regs1 = XITTestRegistry.fromXML(args.reg1)
+ regs2 = XITTestRegistry.fromXML(args.reg2)
+
+ # sort them so searching is simpler
+ regs1.sort()
+ regs2.sort()
+
+ regname = args.regname
+ if regname != None:
+ reg1 = self.find_reg(regname, regs1)
+ if reg1 == None:
+ logging.error("Failed to find '%s' in first registry" % regname)
+ sys.exit(1)
+ reg2 = self.find_reg(regname, regs2)
+ if reg2 == None:
+ logging.error("Failed to find '%s' in second registry" % regname)
+ sys.exit(1)
+ self.compare_registry(reg1, reg2, args.printer);
+ else:
+ failed_regs = []
+ done_regs = []
+ for r1 in regs1:
+ r2 = self.find_reg(r1.name, regs2)
+ if r2 == None:
+ failed_regs.append(r1.name)
+ else:
+ self.compare_registry(r1, r2, args.printer)
+ done_regs.append(r1.name)
+ for r2 in regs2:
+ if r2.name in done_regs:
+ continue
+ failed_regs.append(r2.name)
- self.compare_meta(reg1, reg2)
+ for f in failed_regs:
+ logging.error("Failed to compare '%s'" % f)
- sname_len = 0
- tname_len = 0
- sname_len = max([ len(x[0]) for x in reg1.listTestNames() ]) + 1
- tname_len = max([ len(x[1]) for x in reg1.listTestNames() ]) + 1
+ def find_reg(self, name, reglist):
+ r = [r for r in reglist if r.name == name]
+ return r[0] if len(r) > 0 else None
- format_str = "{0:<4}{1:<%d}{2:<%d}{3:>%d}{4:>%d}" % (sname_len, tname_len, len(reg1.name), len(reg2.name))
+ def compare_registry(self, reg1, reg2, printer):
+ headers = ("Module name", reg1.path, reg2.path)
+ values = self.get_meta_comparison(reg1, reg2)
+ colors = [ Colors.RED if m[1] != m[2] else Colors.DEFAULT for m in values ]
- print format_str.format("Code", "TestSuite", "TestCase", reg1.name, reg2.name)
- print format_str.format("----", "---------", "--------", "-" * len(reg1.name), "-" * len(reg2.name))
+ if len(values):
+ printer.print_values(headers, values, colors = colors, section = reg1.name)
+ headers = ("Code", "TestSuite", "TestCase", reg2.path, reg1.path)
+ values = []
+ colors = []
for suite, test, status in sorted(reg1.listTestNames()):
- self.verify_one_result(reg1.getTest(suite, test), reg2.getTest(suite, test), format_str)
+ c, v = self.verify_one_result(reg1.getTest(suite, test), reg2.getTest(suite, test))
+ colors.append(c)
+ values.append(v)
+ printer.print_values(headers, values, colors = colors)
+
+ def create_registries(self, args):
+ regs = []
+ for r in args.results:
+ regs.append(self.create_registry(r, args))
+
+ r = self.open_new_registry(args)
+ self.write_to_registry(r, regs[0].toXML(regs))
+ self.sync_registry(args, r)
- def create_registry(self, args):
+ def create_registry(self, path, args):
"""Create a new registry XML file based on the test cases in the JUnit file"""
- results_list = JUnitTestResult.fromXML(args.results)
+ results_list = JUnitTestResult.fromXML(path)
results_dict = {}
for r in results_list:
@@ -705,7 +890,7 @@ class XITTestRegistryCLI:
if args.name:
reg_name = args.name[0]
else:
- reg_name = os.path.basename(args.results).split(".xml")[0]
+ reg_name = os.path.basename(path).split(".xml")[0]
registry = XITTestRegistry(reg_name);
registry.date = time.localtime()
@@ -719,56 +904,82 @@ class XITTestRegistryCLI:
testcase = XITTest(suite, r.name, str(r.status).lower())
registry.addTest(testcase);
- r = self.open_new_registry(args)
- self.write_to_registry(r, registry.toXML())
- self.sync_registry(args, r)
+ return registry
def merge_registries(self, args):
"""Merge two registries together"""
if args.add:
self.merge_add_registries(args)
- def merge_add_registries(self, args):
- """Merge registry args.reg2 into regs.arg1, leaving all existing information in reg1 untouched"""
- reg1 = XITTestRegistry.fromXML(args.reg1[0])[0]
- reg2 = XITTestRegistry.fromXML(args.reg2[0])[0]
-
- # merge 2 into 1
+ def merge_registry(self, reg1, reg2):
tests2 = reg2.listTestNames()
for suite, name, status in tests2:
if reg1.getTest(suite, name) == None:
reg1.addTest(reg2.getTest(suite, name))
- self.registry_from_string(args, reg1.toXML());
+
+ def merge_add_registries(self, args):
+ """Merge registry args.reg2 into regs.arg1, leaving all existing information in reg1 untouched"""
+ regs1 = XITTestRegistry.fromXML(args.reg1)
+ regs2 = XITTestRegistry.fromXML(args.reg2)
+
+ if args.regname != None:
+ r1 = self.find_reg(args.regname, regs1)
+ r2 = self.find_reg(args.regname, regs2)
+ if r1 == None and r2 == None:
+ logging.error("Invalid registrys name '%s'" % (args.regname))
+ sys.exit(1)
+
+ if r2 == None:
+ pass # do nothing
+ elif r1 == None:
+ regs1.append(r2)
+ else:
+ self.merge_registry(r1, r2)
+ else:
+ succeeded = []
+ for r1 in regs1:
+ r2 = self.find_reg(r1.name, regs2)
+ if r2 == None:
+ continue
+ self.merge_registry(r1, r2)
+ succeeded.append(r1.name)
+
+ for r2 in regs2:
+ if r2.name in succeeded:
+ continue
+ regs1.append(r2)
+
+ self.registry_from_string(args, r1.toXML(regs1))
def add_bug(self, args):
- registry = self.load_registry(args)
+ registry = self.load_one_registry(args)
testcase = registry.getTest(args.testsuite, args.testcase)
if testcase == None:
- print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase)
+ logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase))
sys.exit(1)
testcase.addBug(XITBug("bugzilla", args.url))
self.registry_from_string(args, registry.toXML())
def rm_bug(self, args):
- registry = self.load_registry(args)
+ registry = self.load_one_registry(args)
testcase = registry.getTest(args.testsuite, args.testcase)
if testcase == None:
- print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase)
+ logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase))
sys.exit(1)
testcase.removeBug(XITBug("bugzilla", args.url))
self.registry_from_string(args, registry.toXML())
def add_fix(self, args, type, text, extra_args = {}):
- registry = self.load_registry(args)
+ registry = self.load_one_registry(args)
testcase = registry.getTest(args.testsuite, args.testcase)
if testcase == None:
- print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase)
+ logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase))
sys.exit(1)
testcase.addFix(XITFix.createFromType(type, text, extra_args))
@@ -785,11 +996,11 @@ class XITTestRegistryCLI:
self.add_fix(args, "rpm", args.rpm)
def rm_fix(self, args, type, text):
- registry = self.load_registry(args)
+ registry = self.load_one_registry(args)
testcase = registry.getTest(args.testsuite, args.testcase)
if testcase == None:
- print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase)
+ logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase))
sys.exit(1)
testcase.removeFix(XITFix.createFromType(type, text))
@@ -802,11 +1013,11 @@ class XITTestRegistryCLI:
self.rm_fix(args, "rpm", args.rpm)
def set_status(self, args):
- registry = self.load_registry(args)
+ registry = self.load_one_registry(args)
testcase = registry.getTest(args.testsuite, args.testcase)
if testcase == None:
- print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase)
+ logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase))
sys.exit(1)
status = { "true" : True,
@@ -817,13 +1028,13 @@ class XITTestRegistryCLI:
try:
testcase.status = status[args.status]
except KeyError:
- print >> sys.stderr, "Invalid status code, allowed are %s" % ",".join(status.keys())
+ logging.error("Invalid status code, allowed are %s" % ",".join(status.keys()))
sys.exit(1)
self.registry_from_string(args, registry.toXML())
def set_date(self, args):
- registry = self.load_registry(args)
+ registry = self.load_one_registry(args)
date = args.date
if date != None:
date = time.strptime(date, "%Y-%m-%d")
@@ -834,7 +1045,7 @@ class XITTestRegistryCLI:
self.registry_from_string(args, registry.toXML())
def set_modversion(self, args):
- registry = self.load_registry(args)
+ registry = self.load_one_registry(args)
name = args.name
version = args.version
type = args.type if args.type else "git"
@@ -851,14 +1062,17 @@ class XITTestRegistryCLI:
"compare the test result with the registry of known test "
"successes/failures.\n")
parser.add_argument("-f", "--file", help="file containing XIT test registry, modified in-place (default: stdin/stdout) ", action="store", required=False)
+ parser.add_argument("-r", "--regname", metavar="registry-name", default=None, help="Work on the named test registry (defaults to first if not given) ", action="store", required=False)
+ parser.add_argument("--output-format", dest="printer", default=XITCLIPrinter(), required=False,
+ action=XITPrinterPicker, help="Pick output format (html, text ,default: text)")
subparsers = parser.add_subparsers(title="Actions", help=None)
list_subparser = subparsers.add_parser("list", help="List all test cases")
list_subparser.set_defaults(func = self.list_tests)
info_subparser = subparsers.add_parser("info", help="Print info about a specific test case")
- info_subparser.add_argument("testsuite", nargs=1, default=None, help="Test Suite name")
- info_subparser.add_argument("testcase", nargs=1, default=None, help="Test Case name")
+ info_subparser.add_argument("testsuite", default=None, help="Test Suite name")
+ info_subparser.add_argument("testcase", default=None, help="Test Case name")
info_subparser.set_defaults(func = self.show_test_info)
verify_subparser = subparsers.add_parser("verify", help="Compare JUnit test results against the registry")
@@ -867,19 +1081,19 @@ class XITTestRegistryCLI:
verify_subparser.set_defaults(func = self.verify_results)
compare_subparser = subparsers.add_parser("compare", help="Compare two test registries")
- compare_subparser.add_argument("reg1", metavar="registry1.xml", nargs=1, help="Registry file no 1")
- compare_subparser.add_argument("reg2", metavar="registry2.xml", nargs=1, help="Registry file no 2")
+ compare_subparser.add_argument("reg1", metavar="registry1.xml", help="Registry file no 1")
+ compare_subparser.add_argument("reg2", metavar="registry2.xml", help="Registry file no 2")
compare_subparser.set_defaults(func = self.compare_registries)
- import_subparser = subparsers.add_parser("create", help="Create new XIT registry from JUnit test results")
- import_subparser.add_argument("results", metavar="results.xml", help="The XML file containing test results")
- import_subparser.add_argument("--name", nargs=1, help="Human-readable name for registry (default: the filename)")
- import_subparser.add_argument("--auto-modversion", metavar="TYPE", nargs=1, help="Try to automatically get module versions for selected modules (default: rpm)")
- import_subparser.set_defaults(func = self.create_registry)
+ create_subparser = subparsers.add_parser("create", help="Create new XIT registry from JUnit test results")
+ create_subparser.add_argument("results", metavar="results.xml", nargs="+", help="The XML file(s) containing test results")
+ create_subparser.add_argument("--name", nargs=1, help="Human-readable name for registry (default: the filename)")
+ create_subparser.add_argument("--auto-modversion", metavar="TYPE", nargs=1, help="Try to automatically get module versions for selected modules (default: rpm)")
+ create_subparser.set_defaults(func = self.create_registries)
merge_subparser = subparsers.add_parser("merge", help="Merge two registries together")
- merge_subparser.add_argument("reg1", metavar="registry1.xml", nargs=1, help="Registry file no 1")
- merge_subparser.add_argument("reg2", metavar="registry2.xml", nargs=1, help="Registry file no 2")
+ merge_subparser.add_argument("reg1", metavar="registry1.xml", help="Registry file no 1")
+ merge_subparser.add_argument("reg2", metavar="registry2.xml", help="Registry file no 2")
merge_subparser.add_argument("--add", default=True, action="store_true", help="Merge new test cases from registry 2 into registry 1, leaving existing test cases unmodified")
merge_subparser.set_defaults(func = self.merge_registries)
@@ -932,22 +1146,45 @@ class XITTestRegistryCLI:
return parser
- def load_registry(self, args):
+ def load_one_registry(self, args):
+ """Load and return a single registry from args.file. If args.name is
+ set, search for a named registry, otherwise return the first"""
+ if args.file == None:
+ logging.error("Reading from stdin")
+ args.file = sys.stdin
+
+ registries = self.load_registry_from_file(args.file)
+ if args.regname != None:
+ for r in registries:
+ if r.name == args.regname:
+ return r
+ logging.error("Failed to find requested registry %s." % args.regname)
+ sys.exit(1)
+ else:
+ logging.warning("Multiple registries found, but no name given. Using first.")
+ return registries[0]
+
+
+ def load_registries(self, args):
+ """Load and return all registries from args.file. If args.name is
+ set, search for a named registry and return only that."""
if args.file == None:
- print >> sys.stderr, "Reading from stdin"
+ logging.error("Reading from stdin")
args.file = sys.stdin
- return self.load_registry_from_file(args.file)
+ if args.regname:
+ return [self.load_one_registry(args)]
+ else:
+ return self.load_registry_from_file(args.file)
def load_registry_from_file(self, path):
+ """Load and return a registry list from the given path."""
registries = XITTestRegistry.fromXML(path)
- if len(registries) > 1:
- print >> sys.stderr, "More than one registry found in input file, this is not supported yet. Using first one only."
- elif len(registries) == 0:
- print >> sys.stderr, "Failed to parse input file."
+ if len(registries) == 0:
+ logging.error("Failed to parse input file.")
sys.exit(1)
- return registries[0]
+ return registries
def write_to_registry(self, f, msg):
print >> f, msg