diff options
author | Peter Hutterer <peter.hutterer@who-t.net> | 2013-02-18 16:41:53 +1000 |
---|---|---|
committer | Peter Hutterer <peter.hutterer@who-t.net> | 2013-02-18 16:41:53 +1000 |
commit | 8baa66c8c846c70a88ff32a3afa989030413ac51 (patch) | |
tree | 08a937f2713a420205ec63251eb70512d0c40430 | |
parent | dcc060bdcb228077d015dd16021b85ffb63d024d (diff) | |
parent | c1b48eeb80a00437dca016b980b71052a58286f8 (diff) |
Merge branch 'multiple-registries'
Conflicts:
registry/server-registry.xml
-rw-r--r-- | HACKING | 27 | ||||
-rw-r--r-- | Makefile.am | 6 | ||||
-rw-r--r-- | common.mk | 2 | ||||
-rw-r--r-- | registry/server-registry.xml | 367 | ||||
-rwxr-xr-x | registry/xit-bug-registry | 3 | ||||
-rwxr-xr-x | registry/xit-bug-registry-test | 97 | ||||
-rw-r--r-- | registry/xit.py | 581 |
7 files changed, 878 insertions, 205 deletions
@@ -222,19 +222,26 @@ track of which tests fail, which is what the bug registry addresses. Often, what really matters is if there are any tests that changed after a fix in the server. To use the bug registry for this task run the following commands. On the __original__ server, run - # Run the grab tests, printing to a JUnit test xml file - ./test/server/grab --gtest_output="xml:grab.xml" - # Create a registry based on the test results - xit-bug-registry create grab.xml > grab-results.xml - # fix server bug - # Re-run grab tests on new server - ./test/server/grab --gtest_output="xml:grab.xml" + # Run all tests. Use -k to ensure all tests are run, even after some have + # failed. + make -k check + # Results end up in $top_builddir/results/latest + # Create a registry based on that. + xit-bug-registry create results/latest/*.xml > before.xml + + # fix bug + # Re-run on new code + make -k check # Compare previous results with new results - xit-bug-registry verify grab.xml < grab-results.xml - + xit-bug-registry create results/latest/*.xml > after.xml + xit-bug-registry compare before.xml after.xml The output will print the test names and the expected vs real outcome plus a -status code to grep for the unexpected. +status code to grep for the unexpected. This will take a while since it runs +all tests. The shortcut is: + # Re-run only the server test + ./server --gtest_output="xml:after.xml" + xit-bug-registry -f before.xml verify after.xml Note that especially test failures need to be treated with caution. An unrelated fix may alter the outcome of a already failing tests (e.g. the diff --git a/Makefile.am b/Makefile.am index d8c364d..57f3282 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,6 +1,10 @@ ACLOCAL_AMFLAGS = -Im4 -SUBDIRS = tests recordings +SUBDIRS = . tests recordings EXTRA_DIST = autogen.sh HACKING 99-ignore-virtual-devices.conf 99-tag-virtual-devices.rules +check-local: + @export DATE=`date +%Y-%m-%d-%H:%M`; \ + mkdir -p $(top_builddir)/results/$$DATE; \ + ln -sfT $$DATE $(top_builddir)/results/latest @@ -1,5 +1,7 @@ include $(top_srcdir)/Makefile-xorg-gtest.am +TESTS_ENVIRONMENT=GTEST_OUTPUT="xml:$(top_builddir)/results/latest/" + AM_CPPFLAGS = \ $(XI_CFLAGS) \ $(XFIXES_CFLAGS) \ diff --git a/registry/server-registry.xml b/registry/server-registry.xml index 52dffd6..da7e0a7 100644 --- a/registry/server-registry.xml +++ b/registry/server-registry.xml @@ -1,5 +1,5 @@ <xit:registries xmlns:xit="http://www.x.org/xorg-integration-testing"> - <xit:registry name="xorg-upstream"> + <xit:registry name="server"> <xit:meta> <xit:date>2013-01-21</xit:date> <xit:moduleversion name="xserver" type="git" repo="git://git.freedesktop.org/git/xorg/xserver">c1602d1c17967bdd4db9db19b3a9c0dfca6a58aa</xit:moduleversion> @@ -661,5 +661,370 @@ </xit:testcase> </xit:testsuite> </xit:registry> + <xit:registry name="input-module-load"> + <xit:meta> + <xit:date>2013-02-05</xit:date> + </xit:meta> + <xit:testsuite name="InputModuleLoadTest"> + <xit:testcase name="CheckForLoadFailure/0" success="true"/> + <xit:testcase name="CheckForLoadFailure/1" success="true"/> + <xit:testcase name="CheckForLoadFailure/10" success="true"/> + <xit:testcase name="CheckForLoadFailure/11" success="false"/> + <xit:testcase name="CheckForLoadFailure/12" success="true"/> + <xit:testcase name="CheckForLoadFailure/13" success="true"/> + <xit:testcase name="CheckForLoadFailure/2" success="true"/> + <xit:testcase name="CheckForLoadFailure/3" success="true"/> + <xit:testcase name="CheckForLoadFailure/4" success="true"/> + <xit:testcase name="CheckForLoadFailure/5" success="true"/> + <xit:testcase name="CheckForLoadFailure/6" success="true"/> + <xit:testcase name="CheckForLoadFailure/7" success="true"/> + <xit:testcase name="CheckForLoadFailure/8" success="true"/> + <xit:testcase name="CheckForLoadFailure/9" success="true"/> + </xit:testsuite> + </xit:registry> + <xit:registry name="kbd-driver"> + <xit:meta> + <xit:date>2013-02-05</xit:date> + </xit:meta> + <xit:testsuite name="KeyboardTest"> + <xit:testcase name="DeviceExists/0" success="true"/> + <xit:testcase name="DeviceExists/1" success="true"/> + <xit:testcase name="DeviceExists/2" success="true"/> + <xit:testcase name="KeyboardLayout/0" success="true"/> + <xit:testcase name="KeyboardLayout/1" success="false"/> + <xit:testcase name="KeyboardLayout/2" success="false"/> + </xit:testsuite> + </xit:registry> + <xit:registry name="legacy-drivers"> + <xit:meta> + <xit:date>2013-02-05</xit:date> + </xit:meta> + <xit:testsuite name="AcecadTest"> + <xit:testcase name="InputDeviceSectionWithOptionDevice" success="true"/> + </xit:testsuite> + <xit:testsuite name="AiptekTest"> + <xit:testcase name="InputDeviceSectionWithType/0" success="true"/> + <xit:testcase name="InputDeviceSectionWithType/1" success="true"/> + <xit:testcase name="InputDeviceSectionWithType/2" success="true"/> + </xit:testsuite> + <xit:testsuite name="ElographicsTest"> + <xit:testcase name="InputDeviceSectionWithOptionDevice" success="true"/> + <xit:testcase name="InvertX" success="true"/> + <xit:testcase name="InvertY" success="true"/> + <xit:testcase name="StylusMovement" success="true"/> + </xit:testsuite> + <xit:testsuite name="LegacyInputDriverTest"> + <xit:testcase name="InputDeviceSectionSimple/0" success="true"/> + <xit:testcase name="InputDeviceSectionSimple/1" success="true"/> + <xit:testcase name="InputDeviceSectionSimple/2" success="true"/> + <xit:testcase name="InputDeviceSectionSimple/3" success="true"/> + <xit:testcase name="InputDeviceSectionSimple/4" success="true"/> + <xit:testcase name="InputDeviceSectionSimple/5" success="true"/> + <xit:testcase name="InputDeviceSectionSimple/6" success="true"/> + </xit:testsuite> + <xit:testsuite name="VoidTest"> + <xit:testcase name="InputDeviceSectionSimple" success="true"/> + </xit:testsuite> + </xit:registry> + <xit:registry name="evdev-driver"> + <xit:meta> + <xit:date>2013-02-05</xit:date> + </xit:meta> + <xit:testsuite name="EvdevAxisLabelTest"> + <xit:testcase name="AbsoluteAxes" success="true"/> + <xit:testcase name="RelAndAbsoluteAxes" success="true"/> + <xit:testcase name="RelativeAxes" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevButtonMappingTest"> + <xit:testcase name="ButtonMapping/0" success="true"/> + <xit:testcase name="ButtonMapping/1" success="true"/> + <xit:testcase name="ButtonMapping/2" success="true"/> + <xit:testcase name="ButtonMapping/3" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevFloatingSlaveTest"> + <xit:testcase name="FloatingDevice" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevInvalidButtonMappingTest"> + <xit:testcase name="InvalidButtonMapping/0" success="true"/> + <xit:testcase name="InvalidButtonMapping/1" success="true"/> + <xit:testcase name="InvalidButtonMapping/2" success="true"/> + <xit:testcase name="InvalidButtonMapping/3" success="true"/> + <xit:testcase name="InvalidButtonMapping/4" success="true"/> + <xit:testcase name="InvalidButtonMapping/5" success="true"/> + <xit:testcase name="InvalidButtonMapping/6" success="true"/> + <xit:testcase name="InvalidButtonMapping/7" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevJoystickTest"> + <xit:testcase name="MTAxesNoButtons" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevMouseTest"> + <xit:testcase name="BtnReleaseMaskOnly" success="true"/> + <xit:testcase name="DevNode" success="true"/> + <xit:testcase name="MiddleButtonEmulation" success="true"/> + <xit:testcase name="ScrollWheel" success="true"/> + <xit:testcase name="SmoothScrolling" success="true"/> + <xit:testcase name="SmoothScrollingAvailable" success="true"/> + <xit:testcase name="TerminateWithButtonDown" success="false"/> + </xit:testsuite> + <xit:testsuite name="EvdevQEMUTabletTest"> + <xit:testcase name="AbsoluteAxesWork" success="true"/> + <xit:testcase name="HasAbsoluteAxes" success="true"/> + <xit:testcase name="HasScrollingAxes" success="true"/> + <xit:testcase name="ScrollingWorks" success="true"/> + <xit:testcase name="SmoothScrollingWorks" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevTouchRotationTest"> + <xit:testcase name="AxisSwapInversion/0" success="true"/> + <xit:testcase name="AxisSwapInversion/1" success="true"/> + <xit:testcase name="AxisSwapInversion/2" success="true"/> + <xit:testcase name="AxisSwapInversion/3" success="true"/> + <xit:testcase name="AxisSwapInversion/4" success="true"/> + <xit:testcase name="AxisSwapInversion/5" success="true"/> + <xit:testcase name="AxisSwapInversion/6" success="true"/> + <xit:testcase name="AxisSwapInversion/7" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevTouchpadTest"> + <xit:testcase name="AxisLabels" success="true"/> + <xit:testcase name="DeviceExists" success="true"/> + <xit:testcase name="PointerMovement" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevXKBConfigRulesTest"> + <xit:testcase name="NoRuleChangeAllowed" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevXKBConfigTest"> + <xit:testcase name="XKBOptionParseTest/0" success="true"/> + <xit:testcase name="XKBOptionParseTest/1" success="true"/> + <xit:testcase name="XKBOptionParseTest/10" success="true"/> + <xit:testcase name="XKBOptionParseTest/11" success="true"/> + <xit:testcase name="XKBOptionParseTest/12" success="true"/> + <xit:testcase name="XKBOptionParseTest/13" success="true"/> + <xit:testcase name="XKBOptionParseTest/14" success="true"/> + <xit:testcase name="XKBOptionParseTest/15" success="true"/> + <xit:testcase name="XKBOptionParseTest/2" success="true"/> + <xit:testcase name="XKBOptionParseTest/3" success="true"/> + <xit:testcase name="XKBOptionParseTest/4" success="true"/> + <xit:testcase name="XKBOptionParseTest/5" success="true"/> + <xit:testcase name="XKBOptionParseTest/6" success="true"/> + <xit:testcase name="XKBOptionParseTest/7" success="true"/> + <xit:testcase name="XKBOptionParseTest/8" success="true"/> + <xit:testcase name="XKBOptionParseTest/9" success="true"/> + </xit:testsuite> + <xit:testsuite name="EvdevXKBTest"> + <xit:testcase name="DeviceExists/0" success="true"/> + <xit:testcase name="DeviceExists/1" success="true"/> + <xit:testcase name="DeviceExists/2" success="true"/> + <xit:testcase name="KeyboardLayout/0" success="true"/> + <xit:testcase name="KeyboardLayout/1" success="true"/> + <xit:testcase name="KeyboardLayout/2" success="true"/> + </xit:testsuite> + </xit:registry> + <xit:registry name="mouse-driver"> + <xit:meta> + <xit:date>2013-02-05</xit:date> + </xit:meta> + <xit:testsuite name="MouseTest"> + <xit:testcase name="BtnPress" success="false"/> + <xit:testcase name="BtnRelease" success="true"/> + <xit:testcase name="DevNode" success="true"/> + <xit:testcase name="Move" success="true"/> + <xit:testcase name="ScrollWheel" success="true"/> + </xit:testsuite> + </xit:registry> + <xit:registry name="synaptics-driver"> + <xit:meta> + <xit:date>2013-02-05</xit:date> + </xit:meta> + <xit:testsuite name="SynapticsClickPad"> + <xit:testcase name="HotPlugSoftButtons" success="false"/> + </xit:testsuite> + <xit:testsuite name="SynapticsClickpadSoftButtonsRuntimeTest"> + <xit:testcase name="SoftButtonsFirst" success="true"/> + <xit:testcase name="SoftButtonsSecond" success="true"/> + </xit:testsuite> + <xit:testsuite name="SynapticsClickpadSoftButtonsTest"> + <xit:testcase name="LeftClick" success="true"/> + <xit:testcase name="LeftClickInDeadArea" success="true"/> + <xit:testcase name="RightClick" success="true"/> + </xit:testsuite> + <xit:testsuite name="SynapticsClickpadSoftButtonsWithAreaTest"> + <xit:testcase name="RightClickInDeadArea" success="true"/> + </xit:testsuite> + <xit:testsuite name="SynapticsClickpadTest"> + <xit:testcase name="ClickpadProperties" success="true"/> + <xit:testcase name="DisableDevice" success="true"/> + <xit:testcase name="DisableDeviceOneFingerDown" success="true"/> + <xit:testcase name="DisableDeviceOneFingerDownAndLift" success="true"/> + <xit:testcase name="DisableDeviceOneFingerResume" success="true"/> + <xit:testcase name="DisableDeviceOneFingerTwoFingersResume" success="true"/> + <xit:testcase name="DisableDeviceTwoFingersDown" success="true"/> + <xit:testcase name="DisableDeviceTwoFingersDownAndLift" success="true"/> + <xit:testcase name="DisableDeviceTwoFingersOneFingerResume" success="true"/> + <xit:testcase name="DisableDeviceTwoFingersResume" success="true"/> + <xit:testcase name="Tap" success="true"/> + <xit:testcase name="VertScrollDown" success="true"/> + <xit:testcase name="VertScrollUp" success="true"/> + </xit:testsuite> + <xit:testsuite name="SynapticsSmoothScrollTest"> + <xit:testcase name="ScrollDelta/0" success="true"/> + <xit:testcase name="ScrollDelta/1" success="true"/> + <xit:testcase name="ScrollDelta/2" success="true"/> + <xit:testcase name="ScrollDelta/3" success="true"/> + <xit:testcase name="ScrollDelta/4" success="true"/> + <xit:testcase name="ScrollDelta/5" success="true"/> + </xit:testsuite> + <xit:testsuite name="SynapticsTest"> + <xit:testcase name="DevicePresent" success="true"/> + <xit:testcase name="ScrollWheel" success="true"/> + <xit:testcase name="SmoothScrollingAvailable" success="true"/> + <xit:testcase name="TapAndDragEvent" success="true"/> + <xit:testcase name="TapEvent" success="true"/> + </xit:testsuite> + <xit:testsuite name="SynapticsWarpTest"> + <xit:testcase name="WarpScaling/0" success="true"/> + <xit:testcase name="WarpScaling/1" success="true"/> + <xit:testcase name="WarpScaling/2" success="true"/> + <xit:testcase name="WarpScaling/3" success="true"/> + <xit:testcase name="WarpScaling/4" success="true"/> + </xit:testsuite> + <xit:testsuite name="SynapticsWarpXineramaTest"> + <xit:testcase name="WarpScaling/0" success="true"/> + <xit:testcase name="WarpScaling/1" success="true"/> + <xit:testcase name="WarpScaling/2" success="true"/> + <xit:testcase name="WarpScaling/3" success="true"/> + <xit:testcase name="WarpScaling/4" success="true"/> + <xit:testcase name="WarpScalingRelative" success="true"/> + </xit:testsuite> + <xit:testsuite name="SynapticsWarpZaphodTest"> + <xit:testcase name="WarpScaling/0" success="true"/> + <xit:testcase name="WarpScaling/1" success="true"/> + <xit:testcase name="WarpScaling/2" success="true"/> + <xit:testcase name="WarpScaling/3" success="true"/> + <xit:testcase name="WarpScaling/4" success="true"/> + </xit:testsuite> + </xit:registry> + <xit:registry name="wacom-driver"> + <xit:meta> + <xit:date>2013-02-05</xit:date> + </xit:meta> + <xit:testsuite name="WacomDriver"> + <xit:testcase name="PrivToolDoubleFree" success="true"/> + </xit:testsuite> + <xit:testsuite name="WacomDriverTest"> + <xit:testcase name="DeviceNames/0" success="true"/> + <xit:testcase name="DeviceNames/1" success="true"/> + <xit:testcase name="DeviceNames/10" success="true"/> + <xit:testcase name="DeviceNames/11" success="true"/> + <xit:testcase name="DeviceNames/12" success="false"/> + <xit:testcase name="DeviceNames/13" success="false"/> + <xit:testcase name="DeviceNames/14" success="true"/> + <xit:testcase name="DeviceNames/2" success="true"/> + <xit:testcase name="DeviceNames/3" success="true"/> + <xit:testcase name="DeviceNames/4" success="true"/> + <xit:testcase name="DeviceNames/5" success="true"/> + <xit:testcase name="DeviceNames/6" success="true"/> + <xit:testcase name="DeviceNames/7" success="true"/> + <xit:testcase name="DeviceNames/8" success="true"/> + <xit:testcase name="DeviceNames/9" success="false"/> + <xit:testcase name="DeviceType/0" success="true"/> + <xit:testcase name="DeviceType/1" success="true"/> + <xit:testcase name="DeviceType/10" success="true"/> + <xit:testcase name="DeviceType/11" success="true"/> + <xit:testcase name="DeviceType/12" success="false"/> + <xit:testcase name="DeviceType/13" success="true"/> + <xit:testcase name="DeviceType/14" success="true"/> + <xit:testcase name="DeviceType/2" success="true"/> + <xit:testcase name="DeviceType/3" success="true"/> + <xit:testcase name="DeviceType/4" success="true"/> + <xit:testcase name="DeviceType/5" success="true"/> + <xit:testcase name="DeviceType/6" success="true"/> + <xit:testcase name="DeviceType/7" success="true"/> + <xit:testcase name="DeviceType/8" success="true"/> + <xit:testcase name="DeviceType/9" success="true"/> + <xit:testcase name="Rotation/0" success="true"/> + <xit:testcase name="Rotation/1" success="true"/> + <xit:testcase name="Rotation/10" success="true"/> + <xit:testcase name="Rotation/11" success="true"/> + <xit:testcase name="Rotation/12" success="false"/> + <xit:testcase name="Rotation/13" success="false"/> + <xit:testcase name="Rotation/14" success="true"/> + <xit:testcase name="Rotation/2" success="true"/> + <xit:testcase name="Rotation/3" success="true"/> + <xit:testcase name="Rotation/4" success="true"/> + <xit:testcase name="Rotation/5" success="true"/> + <xit:testcase name="Rotation/6" success="true"/> + <xit:testcase name="Rotation/7" success="true"/> + <xit:testcase name="Rotation/8" success="true"/> + <xit:testcase name="Rotation/9" success="false"/> + </xit:testsuite> + <xit:testsuite name="WacomHoveringTest"> + <xit:testcase name="HoveringTest" success="false"/> + </xit:testsuite> + <xit:testsuite name="WacomLensCursorTest"> + <xit:testcase name="CursorMove/0" success="true"/> + </xit:testsuite> + <xit:testsuite name="WacomMatrixTest"> + <xit:testcase name="DevicePresent" success="true"/> + <xit:testcase name="InputMatrix" success="false"/> + </xit:testsuite> + <xit:testsuite name="WacomPropertyTest"> + <xit:testcase name="Button1DoubleMiddleClick" success="false"/> + <xit:testcase name="ButtonActionHighButtonValue" success="false"/> + <xit:testcase name="ButtonActionInvalidFormat" success="false"/> + <xit:testcase name="ButtonActionInvalidType" success="false"/> + <xit:testcase name="ButtonActionKeyPress" success="false"/> + <xit:testcase name="ButtonActionKeyPressRelease" success="false"/> + <xit:testcase name="ButtonActionPropertiesPresent" success="false"/> + <xit:testcase name="ButtonActionPropertySetToNone" success="false"/> + <xit:testcase name="ButtonActionPropertyUnset" success="false"/> + </xit:testsuite> + <xit:testsuite name="WacomToolProximityTest"> + <xit:testcase name="ToolMovesOnProximity/0" success="true"/> + <xit:testcase name="ToolMovesOnProximity/1" success="true"/> + <xit:testcase name="ToolMovesOnProximity/2" success="true"/> + <xit:testcase name="ToolMovesOnProximity/3" success="true"/> + <xit:testcase name="ToolMovesOnProximity/4" success="true"/> + <xit:testcase name="ToolMovesOnProximity/5" success="true"/> + <xit:testcase name="ToolMovesOnProximity/6" success="true"/> + </xit:testsuite> + </xit:registry> + <xit:registry name="libXi"> + <xit:meta> + <xit:date>2013-02-05</xit:date> + </xit:meta> + <xit:testsuite name="libXiTest"> + <xit:testcase name="DisplayNotGarbage" success="true"/> + </xit:testsuite> + <xit:testsuite name="libXiTouchTest"> + <xit:testcase name="CopyRawTouchEvent" success="true"/> + </xit:testsuite> + </xit:registry> + <xit:registry name="video-module-load"> + <xit:meta> + <xit:date>2013-02-05</xit:date> + </xit:meta> + <xit:testsuite name="VideoModuleLoadTest"> + <xit:testcase name="CheckForLoadFailure/0" success="false"/> + <xit:testcase name="CheckForLoadFailure/1" success="false"/> + <xit:testcase name="CheckForLoadFailure/10" success="false"/> + <xit:testcase name="CheckForLoadFailure/11" success="false"/> + <xit:testcase name="CheckForLoadFailure/12" success="false"/> + <xit:testcase name="CheckForLoadFailure/13" success="false"/> + <xit:testcase name="CheckForLoadFailure/14" success="false"/> + <xit:testcase name="CheckForLoadFailure/15" success="false"/> + <xit:testcase name="CheckForLoadFailure/16" success="false"/> + <xit:testcase name="CheckForLoadFailure/17" success="false"/> + <xit:testcase name="CheckForLoadFailure/18" success="false"/> + <xit:testcase name="CheckForLoadFailure/19" success="false"/> + <xit:testcase name="CheckForLoadFailure/2" success="true"/> + <xit:testcase name="CheckForLoadFailure/20" success="false"/> + <xit:testcase name="CheckForLoadFailure/3" success="false"/> + <xit:testcase name="CheckForLoadFailure/4" success="false"/> + <xit:testcase name="CheckForLoadFailure/5" success="false"/> + <xit:testcase name="CheckForLoadFailure/6" success="true"/> + <xit:testcase name="CheckForLoadFailure/7" success="false"/> + <xit:testcase name="CheckForLoadFailure/8" success="false"/> + <xit:testcase name="CheckForLoadFailure/9" success="false"/> + </xit:testsuite> + </xit:registry> </xit:registries> diff --git a/registry/xit-bug-registry b/registry/xit-bug-registry index 1eddc0e..8e90fd6 100755 --- a/registry/xit-bug-registry +++ b/registry/xit-bug-registry @@ -38,6 +38,9 @@ # import xit +import logging + +logging.basicConfig(level=logging.DEBUG) cli = xit.XITTestRegistryCLI() cli.run() diff --git a/registry/xit-bug-registry-test b/registry/xit-bug-registry-test index 0d985f1..3087189 100755 --- a/registry/xit-bug-registry-test +++ b/registry/xit-bug-registry-test @@ -9,6 +9,9 @@ import time import os import shutil +REGISTRY_SOURCE_FILE = "server-registry.xml" +REGISTRY_FILENAME = "/tmp/tmp.registry.xml" + class TestXITBug(unittest.TestCase): def test_defaults(self): @@ -265,6 +268,20 @@ class TestXITTestRegistry(unittest.TestCase): return reg2 + def to_from_xml_multiple(self, reg): + xml = reg.toXML() + + f = tempfile.TemporaryFile() + f.write(xml) + f.seek(0) + + reg2 = xit.XITTestRegistry.fromXML(f) + xml2 = reg2[0].toXML(reg2) + + self.assertEqual(xml, xml2) + + return reg2 + def test_to_from_xml(self): reg = xit.XITTestRegistry() self.to_from_xml(reg); @@ -288,27 +305,47 @@ class TestXITTestRegistry(unittest.TestCase): for mod, mod2 in zip(reg.moduleversions, reg2.moduleversions): self.assertEqual(mod, mod2) + def test_multiple_registries(self): + regs = xit.XITTestRegistry.fromXML(REGISTRY_SOURCE_FILE) + self.assertGreater(len(regs), 1) + for r in regs: + self.assertGreater(len(r.name), 0) + self.assertGreater(len(r.tests), 0) + + def test_multiple_registries_write_load(self): + regs = xit.XITTestRegistry.fromXML(REGISTRY_SOURCE_FILE) + xml = regs[0].toXML(regs) + + f = tempfile.TemporaryFile() + f.write(xml) + f.seek(0) + + regs2 = xit.XITTestRegistry.fromXML(f) + self.assertEqual(len(regs), len(regs2)) + xml2 = regs2[0].toXML(regs2) + + self.assertEqual(xml, xml2) class TestXITTestRegistryCLI(unittest.TestCase): + EXIT_GENERAL_FAILURE = 1 EXIT_TOO_FEW_ARGS_ERROR_CODE = 2 - REGISTRY_SOURCE_FILE = "server-registry.xml" - REGISTRY_FILENAME = "/tmp/tmp.registry.xml" - RESULTS_FILE = "../server-results.xml" + RESULTS_FILE = "../server.xml" def setUp(self): """Make sure server-registry.xml is present, which it should be in this directory""" - self.assertTrue(os.path.exists(self.REGISTRY_SOURCE_FILE)) + self.assertTrue(os.path.exists(REGISTRY_SOURCE_FILE)) self.assertTrue(os.path.exists(self.RESULTS_FILE)) - shutil.copyfile(self.REGISTRY_SOURCE_FILE, self.REGISTRY_FILENAME) + shutil.copyfile(REGISTRY_SOURCE_FILE, REGISTRY_FILENAME) self.cli = xit.XITTestRegistryCLI() - self.registry = self.cli.load_registry_from_file(self.REGISTRY_FILENAME) + # FIXME: for now only work on the first one + self.registry = self.cli.load_registry_from_file(REGISTRY_FILENAME)[0] # default args - self.args = ["-f", self.REGISTRY_FILENAME] + self.args = ["-f", REGISTRY_FILENAME] def test_load_registry(self): noargs = "" @@ -324,10 +361,24 @@ class TestXITTestRegistryCLI(unittest.TestCase): self.cli.run(self.args) self.assertEquals(e.exception.code, self.EXIT_TOO_FEW_ARGS_ERROR_CODE) + # with a registry that doesn't exist + with self.assertRaises(SystemExit) as e: + self.cli.run(self.args + ["-r", "doesnotexist"]) + self.assertEquals(e.exception.code, self.EXIT_TOO_FEW_ARGS_ERROR_CODE); + + # with a registry that does exist + with self.assertRaises(SystemExit) as e: + self.cli.run(self.args + ["-r", "server"]) + self.assertEquals(e.exception.code, self.EXIT_TOO_FEW_ARGS_ERROR_CODE); + def test_list_registry(self): - args = self.args - args.append("list") - self.cli.run(args) + self.cli.run(self.args + ["list"]) + + with self.assertRaises(SystemExit) as e: + self.cli.run(self.args + ["-r", "doesnotexist", "list"]) + self.assertEquals(e.exception.code, self.EXIT_GENERAL_FAILURE); + + self.cli.run(self.args + ["-r", "server", "list"]) def test_show_info(self): args = self.args @@ -345,14 +396,17 @@ class TestXITTestRegistryCLI(unittest.TestCase): self.cli.run(args + [suite] + [test]) def test_compare(self): - args = self.args - args += ["compare", self.REGISTRY_FILENAME, self.REGISTRY_FILENAME] - self.cli.run(args) + self.cli.run(self.args + ["compare", REGISTRY_FILENAME, REGISTRY_FILENAME]) + self.cli.run(self.args + ["-r", "server", "compare", REGISTRY_FILENAME, REGISTRY_FILENAME]) + + with self.assertRaises(SystemExit) as e: + self.cli.run(self.args + ["-r", "doesnotexist", "compare", REGISTRY_FILENAME, REGISTRY_FILENAME]) + self.assertEquals(e.exception.code, self.EXIT_GENERAL_FAILURE); def test_merge(self): - args = self.args - args += ["merge", self.REGISTRY_FILENAME, self.REGISTRY_FILENAME] - self.cli.run(args) + + self.cli.run(self.args + ["-r", "server", "merge", REGISTRY_FILENAME, REGISTRY_FILENAME]) + self.cli.run(self.args + ["merge", REGISTRY_FILENAME, REGISTRY_FILENAME]) def test_edit(self): args = self.args @@ -398,12 +452,13 @@ class TestXITTestRegistryCLI(unittest.TestCase): self.cli.run(args + ["--name", "somename", self.RESULTS_FILE]) self.cli.run(args + ["--auto-modversion", "rpm", self.RESULTS_FILE]) - def test_create(self): - args = self.args - args.append("verify") + def test_verify(self): + self.cli.run(self.args + ["verify", self.RESULTS_FILE]) + self.cli.run(self.args + ["verify", "--check-all", self.RESULTS_FILE]) - self.cli.run(args + [self.RESULTS_FILE]) - self.cli.run(args + ["--check-all", self.RESULTS_FILE]) + with self.assertRaises(SystemExit) as e: + self.cli.run(self.args + ["-r", "evdev", "verify", "--check-all", self.RESULTS_FILE]) + self.assertEquals(e.exception.code, self.EXIT_GENERAL_FAILURE); if __name__ == '__main__': unittest.main() diff --git a/registry/xit.py b/registry/xit.py index cd0d49a..b37465b 100644 --- a/registry/xit.py +++ b/registry/xit.py @@ -45,6 +45,7 @@ from lxml import objectify import lxml.etree import shutil import subprocess +import logging DEFAULT_MODULES = ["xorg-x11-server-Xorg", "xorg-x11-drv-evdev", @@ -53,8 +54,11 @@ DEFAULT_MODULES = ["xorg-x11-server-Xorg", "xorg-x11-drv-mouse", "xorg-x11-drv-keyboard"] -def debug(msg): - print >> sys.stderr, msg +class Colors: + DEFAULT = 0 + RED = 1 + GREEN = 2 + BLUE = 3 XMLNS = "http://www.x.org/xorg-integration-testing" def xmlns_tag(tag, ns=XMLNS): @@ -79,6 +83,14 @@ class termcolors: BLUE = '\033[1;34m' @classmethod + def color(self, color): + colors = { Colors.DEFAULT : termcolors.DEFAULT, + Colors.RED : termcolors.RED, + Colors.GREEN : termcolors.GREEN, + Colors.BLUE : termcolors.BLUE } + return colors[color] + + @classmethod def disable(self): self.DEFAULT = '' self.RED = '' @@ -88,12 +100,13 @@ class termcolors: class XITTestRegistry: """Central class keeping a set of test cases and their results""" - def __init__(self, name="", test_cases = []): + def __init__(self, name="", test_cases = [], path="stdin"): """Initialise with a registry name and a list of """ self.tests = self._from_list(test_cases) self.name = name self.date = time.localtime() self.moduleversions = [] + self.path = path @classmethod def fromXML(self, filename): @@ -101,7 +114,7 @@ class XITTestRegistry: registries = objectify.parse(filename).getroot() regs = [] for registry in registries.iterchildren(tag=xmlns_tag("registry")): - reg = XITTestRegistry(name=registry.attrib["name"]) + reg = XITTestRegistry(name=registry.attrib["name"], path=filename) for meta in registry.iterchildren(tag=xmlns_tag("meta")): date = meta.find(xmlns_tag("date")) @@ -113,7 +126,11 @@ class XITTestRegistry: type = modversion.attrib["type"] except KeyError: type = "git" - mv = XITModuleVersion(modversion.attrib["name"], modversion.text, type) + try: + repo = modversion.attrib["repo"] + except KeyError: + repo = None + mv = XITModuleVersion(modversion.attrib["name"], modversion.text, type, repo) reg.moduleversions.append(mv) for suite in registry.iterchildren(tag=xmlns_tag("testsuite")): @@ -143,56 +160,67 @@ class XITTestRegistry: regs.append(reg) return regs - def toXML(self): - """Generate XML output from this registry and return it""" + def toXML(self, others=[]): + """Generate XML output from this registry and return it. If others + is a list, the resulting XML file contains all of the registries + provided including self. Otherwise, only self is written out.""" NSMAP = { "xit" : "http://www.x.org/xorg-integration-testing" } E = objectify.ElementMaker(annotate = False, namespace = NSMAP['xit'], nsmap = NSMAP) - xit_registries = E.registries() - xit_registry = E.registry() - xit_registry.set("name", self.name) - xit_registries.append(xit_registry) - - xit_meta = E.meta() - xit_registry.append(xit_meta) - xit_date = E.date(time.strftime("%Y-%m-%d", self.date)) - xit_meta.append(xit_date) - - for modversion in sorted(self.moduleversions): - xit_modversion = E.moduleversion(modversion.version) - xit_modversion.set("name", modversion.module) - xit_modversion.set("type", modversion.type) - xit_meta.append(xit_modversion) + xit_registries = E.registries() - for suite_name, suite in sorted(self.tests.iteritems()): - xit_suite = E.testsuite() - xit_suite.set("name", suite_name) - for name, test in sorted(suite.iteritems()): - xit_testcase = E.testcase() - xit_testcase.set("name", test.name) - xit_testcase.set("success", str(test.status).lower()) - - for bug in test.getBugs(): - xit_bug = E.bug(bug.url) - xit_bug.set("type", bug.type) - xit_testcase.append(xit_bug) - - for fix in test.getFixes(): - xit_fix = E.fix(fix.text) - xit_fix.set("type", fix.type); - for arg, value in fix.extra_args.iteritems(): - xit_fix.set(arg, value) - xit_testcase.append(xit_fix) - - for info in test.getInfo(): - xit_info = E.testinfo(info.text) - xit_info.set("type", info.type) - xit_testcase.append(xit_info) - - xit_suite.append(xit_testcase) - xit_registry.append(xit_suite) + if len(others) == 0: + others = [self] # yikes + + for r in others: + xit_registry = E.registry() + xit_registry.set("name", r.name) + xit_registries.append(xit_registry) + + xit_meta = E.meta() + xit_registry.append(xit_meta) + + xit_date = E.date(time.strftime("%Y-%m-%d", r.date)) + xit_meta.append(xit_date) + + for modversion in sorted(r.moduleversions): + xit_modversion = E.moduleversion(modversion.version) + xit_modversion.set("name", modversion.module) + xit_modversion.set("type", modversion.type) + if modversion.repo: + xit_modversion.set("repo", modversion.repo) + + xit_meta.append(xit_modversion) + + for suite_name, suite in sorted(r.tests.iteritems()): + xit_suite = E.testsuite() + xit_suite.set("name", suite_name) + for name, test in sorted(suite.iteritems()): + xit_testcase = E.testcase() + xit_testcase.set("name", test.name) + xit_testcase.set("success", str(test.status).lower()) + + for bug in test.getBugs(): + xit_bug = E.bug(bug.url) + xit_bug.set("type", bug.type) + xit_testcase.append(xit_bug) + + for fix in test.getFixes(): + xit_fix = E.fix(fix.text) + xit_fix.set("type", fix.type); + for arg, value in fix.extra_args.iteritems(): + xit_fix.set(arg, value) + xit_testcase.append(xit_fix) + + for info in test.getInfo(): + xit_info = E.testinfo(info.text) + xit_info.set("type", info.type) + xit_testcase.append(xit_info) + + xit_suite.append(xit_testcase) + xit_registry.append(xit_suite) lxml.etree.cleanup_namespaces(xit_registries) return lxml.etree.tostring(xit_registries, pretty_print=True) @@ -227,6 +255,8 @@ class XITTestRegistry: self.tests[test.suite] = {} self.tests[test.suite][test.name] = test + def __cmp__(self, other): + return cmp(self.name, other) class XITTestCase: """Represents one single test case, comprised of test suite name and test case name. @@ -462,9 +492,10 @@ class XITInfoURL(XITInfo): class XITModuleVersion: """Represents a module version of a particular component.""" - def __init__(self, module, version, type = "git"): + def __init__(self, module, version, type = "git", repo=None): self.module = module self.version = version + self.repo = repo self.type = type def __str__(self): @@ -548,46 +579,169 @@ class JUnitTestFailure: return self.message +class XITCLIPrinter: + def __init__(self): + pass + + def separator(self, text): + print ":" * 20 + " {:<30} ".format(text) + ":" * 58 + + def print_values(self, headers, values, colors = None, section=None): + if len(headers) != len(values[0]): + logging.error("Mismatched header/values tuples. Refusing to print") + return + + if section != None: + self.separator(section) + + cw = [] # column width + separators = [] + for header in headers: + cw.append(len(header)) + separators.append("-" * len(header)) + + for val in values: + for i in range(len(val)): + cw[i] = max(cw[i], len(str(val[i]))) + + format_str = "" + for idx, w in zip(range(len(cw)), cw): + format_str += "{%d:<%d}" % (idx, w + 1) + + print format_str.format(*headers) + print format_str.format(*separators) + + if not colors: + colors = [Colors.DEFAULT] * len(values) + + for color, val in zip(colors, values): + print termcolors.color(color) + format_str.format(*val) + termcolors.DEFAULT + + +class XITPrinterPicker(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + if values == "html": + printer = XITHTMLPrinter() + else: + printer = XITCLIPrinter() + + setattr(namespace, "printer", printer) + + +class XITHTMLPrinter: + def __init__(self): + self.print_html_header() + + def __del__(self): + self.print_html_footer() + + def print_html_header(self): + print """<html> <head> + <style> + body{font-family:sans-serif;} + .values {border-collapse:collapse;font-size:80%;} + .values td, th {padding:0.5em 1em; border:1px solid #ccc; text-align:left;} values {background-color:#eee;text-align:left; font-size:80%;} + .red { background-color:#ee0000; text: #000000; } + </style> + </head><body> + """ + + def print_html_footer(self): + print """</body></html> + """ + + def print_table_header(self): + print """<table class=\"values\">""" + + def print_table_footer(self): + print """</table>""" + + def print_values(self, headers, values, colors = None, section = None): + if section: + print "<h2>%s</h2>" % section + self.print_table_header() + + format_str = "<tr>" + format_str += "<th>{}</th>" * len(headers) + format_str += "</tr>" + print format_str.format(*headers) + + if not colors: + colors = [Colors.DEFAULT] * len(values) + + for color, val in zip(colors, values): + if color == Colors.RED: + format_str = "<tr class=\"red\">" + else: + format_str = "<tr>" + + format_str += "<td>{}</td>" * len(val) + format_str += "</tr>" + print format_str.format(*val) + + self.print_table_footer() + class XITTestRegistryCLI: + """Command-line interface to the registry""" + def list_tests(self, args): """List all tests, showing test name and expected status""" - registry = self.load_registry(args) - all_tests = registry.listTestNames() - all_tests.insert(0, ("TestSuite", "TestCase", "Success")) - all_tests.insert(1, ("---------", "--------", "-------")) - for suite, test, status in all_tests: - print "{:<50}{:<50}{:>10}".format(suite, test, str(status)) + registries = self.load_registries(args) + for r in registries: + all_tests = r.listTestNames() + + headers = ("TestSuite", "TestCase", "Success") + values = [(suite, test, str(status)) for suite, test, status in all_tests] + colors = [Colors.RED if not status else Colors.DEFAULT for (suite, test, status) in all_tests] + + args.printer.print_values(headers, values, colors = colors, section = r.name) def show_test_info(self, args): """Show all information about a given XIT registry test case""" - registry = self.load_registry(args) - test = registry.getTest(args.testsuite[0], args.testcase[0]) + registry = self.load_one_registry(args) + test = registry.getTest(args.testsuite, args.testcase) if test != None: print str(test) + else: + logging.error("Unable to find test %s.%s\n", args.testsuite, args.testcase) - def verify_one_result(self, test, result, format_str): + def verify_one_result(self, test, result): """Verify the test result given against the registry. Prints a status message comprising full test name, expected outcome, actual outcome and a grep-able bit to indicate which way the outcome differs""" - color = termcolors.DEFAULT - if test != None: + color = Colors.DEFAULT + + if test != None and result != None: + suite = test.suite + name = test.name + status = str(result.status).lower() expected_status = str(test.status).lower() if str(test.status).lower() != str(result.status).lower(): status_match = "XX" if result.status == True: - color = termcolors.GREEN + color = Colors.GREEN else: - color = termcolors.RED + color = Colors.RED else: status_match = "++" if test.status else "--" + elif test != None and result == None: + suite = test.suite + name = test.name + status_match = "??" + expected_status = str(test.status).lower() + status = "" + color = Colors.BLUE else: + suite = result.suite + name = result.name + status = str(result.status).lower() expected_status = "" status_match = "??" - color = termcolors.BLUE + color = Colors.BLUE - print color + format_str.format(status_match, result.suite, result.name, str(result.status).lower(), expected_status) + termcolors.DEFAULT + return (color, (status_match, suite, name, status, expected_status)) def check_installed_rpms(self, registry): @@ -601,26 +755,38 @@ class XITTestRegistryCLI: def verify_results(self, args): - """Verify a JUnit test result against the XIT test registry""" - registry = self.load_registry(args) - results = JUnitTestResult.fromXML(args.results) - - sname_len = 0 - tname_len = 0 + """Verify a JUnit test result against the XIT test registry. + If a registry name is given, compare to that. If none is given + and there is more than one registry, search for one named the + same as the results file. Otherwise, bail out.""" + registry = None + registries = self.load_registries(args) + if not args.regname: + regname = os.path.basename(args.results).split(".xml")[0] + for r in registries: + if r.name == regname: + registry = r + break + if registry == None: + logging.error("Failed to match %s with a registry." % args.results) + sys.exit(1) + else: + registry = registries[0] - sname_len = max([ len(x.suite) for x in results ]) + 1 - tname_len = max([ len(x.name) for x in results ]) + 1 + results = JUnitTestResult.fromXML(args.results) if args.check_all: self.check_installed_rpms(registry) - format_str = "{0:<5}{1:<%d}{2:<%d}{3:>10}{4:>10}" % (sname_len, tname_len) - print format_str.format("Code", "TestSuite", "TestCase", "Result", "Expected") - print format_str.format("----", "---------", "--------", "------", "--------") - + headers = ("Code", "TestSuite", "TestCase", "Result", "Expected") + values = [] + colors = [] for result in sorted(results): - self.verify_one_result(registry.getTest(result.suite, result.name), result, format_str) + c, v = self.verify_one_result(registry.getTest(result.suite, result.name), result) + colors.append(c) + values.append(v) + args.printer.print_values(headers, values, colors = colors, section = registry.name) def print_modversions(self, name, v1, v2, use_colors=True): if use_colors and v1 != v2: @@ -630,71 +796,90 @@ class XITTestRegistryCLI: format_str = "{0:<30} {1:<50} {2:<50}" print color + format_str.format(name, v1, v2) + termcolors.DEFAULT - def compare_meta(self, reg1, reg2): - self.print_modversions("Module name", reg1.name, reg2.name, False) - self.print_modversions("-----------", "----------", "----------", False) - + def get_meta_comparison(self, reg1, reg2): + modules = [ r.module for r in reg1.moduleversions] + modules.extend(r.module for r in reg2.moduleversions if r.name not in modules) - r1_iter = iter(sorted(reg1.moduleversions)) - r2_iter = iter(sorted(reg2.moduleversions)) - - try: - r1_modversion = r1_iter.next() - r2_modversion = r2_iter.next() - while True: - rc = cmp(r1_modversion, r2_modversion) - if rc == 0: - self.print_modversions(r1_modversion.module, r1_modversion.version, r2_modversion.version) - r1_modversion = r1_iter.next() - r2_modversion = r2_iter.next() - elif rc > 0: - self.print_modversions(r2_modversion.module, "???", r2_modversion.version) - r2_modversion = r2_iter.next() - elif rc < 0: - self.print_modversions(r1_modversion.module, r1_modversion.version, "???") - r1_modversion = r1_iter.next() - except StopIteration: - pass - - try: - while True: - r1_modversion = r1_iter.next() - self.print_modversions(r1_modversion.module, r1_modversion.version, "???") - except StopIteration: - pass - try: - while True: - r2_modversion = r2_iter.next() - self.print_modversions(r2_modversion.module, "???", r2_modversion.version) - except StopIteration: - pass + ret = [] + for m in modules: + r1 = "".join([ mr1.version for mr1 in reg1.moduleversions if mr1.module == m ]) + r2 = "".join([ mr2.version for mr2 in reg2.moduleversions if mr2.module == m ]) + ret.append((m, r1, r2)) - print + return ret def compare_registries(self, args): - reg1 = XITTestRegistry.fromXML(args.reg1[0])[0] - reg2 = XITTestRegistry.fromXML(args.reg2[0])[0] + regs1 = XITTestRegistry.fromXML(args.reg1) + regs2 = XITTestRegistry.fromXML(args.reg2) + + # sort them so searching is simpler + regs1.sort() + regs2.sort() + + regname = args.regname + if regname != None: + reg1 = self.find_reg(regname, regs1) + if reg1 == None: + logging.error("Failed to find '%s' in first registry" % regname) + sys.exit(1) + reg2 = self.find_reg(regname, regs2) + if reg2 == None: + logging.error("Failed to find '%s' in second registry" % regname) + sys.exit(1) + self.compare_registry(reg1, reg2, args.printer); + else: + failed_regs = [] + done_regs = [] + for r1 in regs1: + r2 = self.find_reg(r1.name, regs2) + if r2 == None: + failed_regs.append(r1.name) + else: + self.compare_registry(r1, r2, args.printer) + done_regs.append(r1.name) + for r2 in regs2: + if r2.name in done_regs: + continue + failed_regs.append(r2.name) - self.compare_meta(reg1, reg2) + for f in failed_regs: + logging.error("Failed to compare '%s'" % f) - sname_len = 0 - tname_len = 0 - sname_len = max([ len(x[0]) for x in reg1.listTestNames() ]) + 1 - tname_len = max([ len(x[1]) for x in reg1.listTestNames() ]) + 1 + def find_reg(self, name, reglist): + r = [r for r in reglist if r.name == name] + return r[0] if len(r) > 0 else None - format_str = "{0:<4}{1:<%d}{2:<%d}{3:>%d}{4:>%d}" % (sname_len, tname_len, len(reg1.name), len(reg2.name)) + def compare_registry(self, reg1, reg2, printer): + headers = ("Module name", reg1.path, reg2.path) + values = self.get_meta_comparison(reg1, reg2) + colors = [ Colors.RED if m[1] != m[2] else Colors.DEFAULT for m in values ] - print format_str.format("Code", "TestSuite", "TestCase", reg1.name, reg2.name) - print format_str.format("----", "---------", "--------", "-" * len(reg1.name), "-" * len(reg2.name)) + if len(values): + printer.print_values(headers, values, colors = colors, section = reg1.name) + headers = ("Code", "TestSuite", "TestCase", reg2.path, reg1.path) + values = [] + colors = [] for suite, test, status in sorted(reg1.listTestNames()): - self.verify_one_result(reg1.getTest(suite, test), reg2.getTest(suite, test), format_str) + c, v = self.verify_one_result(reg1.getTest(suite, test), reg2.getTest(suite, test)) + colors.append(c) + values.append(v) + printer.print_values(headers, values, colors = colors) + + def create_registries(self, args): + regs = [] + for r in args.results: + regs.append(self.create_registry(r, args)) + + r = self.open_new_registry(args) + self.write_to_registry(r, regs[0].toXML(regs)) + self.sync_registry(args, r) - def create_registry(self, args): + def create_registry(self, path, args): """Create a new registry XML file based on the test cases in the JUnit file""" - results_list = JUnitTestResult.fromXML(args.results) + results_list = JUnitTestResult.fromXML(path) results_dict = {} for r in results_list: @@ -705,7 +890,7 @@ class XITTestRegistryCLI: if args.name: reg_name = args.name[0] else: - reg_name = os.path.basename(args.results).split(".xml")[0] + reg_name = os.path.basename(path).split(".xml")[0] registry = XITTestRegistry(reg_name); registry.date = time.localtime() @@ -719,56 +904,82 @@ class XITTestRegistryCLI: testcase = XITTest(suite, r.name, str(r.status).lower()) registry.addTest(testcase); - r = self.open_new_registry(args) - self.write_to_registry(r, registry.toXML()) - self.sync_registry(args, r) + return registry def merge_registries(self, args): """Merge two registries together""" if args.add: self.merge_add_registries(args) - def merge_add_registries(self, args): - """Merge registry args.reg2 into regs.arg1, leaving all existing information in reg1 untouched""" - reg1 = XITTestRegistry.fromXML(args.reg1[0])[0] - reg2 = XITTestRegistry.fromXML(args.reg2[0])[0] - - # merge 2 into 1 + def merge_registry(self, reg1, reg2): tests2 = reg2.listTestNames() for suite, name, status in tests2: if reg1.getTest(suite, name) == None: reg1.addTest(reg2.getTest(suite, name)) - self.registry_from_string(args, reg1.toXML()); + + def merge_add_registries(self, args): + """Merge registry args.reg2 into regs.arg1, leaving all existing information in reg1 untouched""" + regs1 = XITTestRegistry.fromXML(args.reg1) + regs2 = XITTestRegistry.fromXML(args.reg2) + + if args.regname != None: + r1 = self.find_reg(args.regname, regs1) + r2 = self.find_reg(args.regname, regs2) + if r1 == None and r2 == None: + logging.error("Invalid registrys name '%s'" % (args.regname)) + sys.exit(1) + + if r2 == None: + pass # do nothing + elif r1 == None: + regs1.append(r2) + else: + self.merge_registry(r1, r2) + else: + succeeded = [] + for r1 in regs1: + r2 = self.find_reg(r1.name, regs2) + if r2 == None: + continue + self.merge_registry(r1, r2) + succeeded.append(r1.name) + + for r2 in regs2: + if r2.name in succeeded: + continue + regs1.append(r2) + + self.registry_from_string(args, r1.toXML(regs1)) def add_bug(self, args): - registry = self.load_registry(args) + registry = self.load_one_registry(args) testcase = registry.getTest(args.testsuite, args.testcase) if testcase == None: - print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase) + logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase)) sys.exit(1) testcase.addBug(XITBug("bugzilla", args.url)) self.registry_from_string(args, registry.toXML()) def rm_bug(self, args): - registry = self.load_registry(args) + registry = self.load_one_registry(args) testcase = registry.getTest(args.testsuite, args.testcase) if testcase == None: - print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase) + logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase)) sys.exit(1) testcase.removeBug(XITBug("bugzilla", args.url)) self.registry_from_string(args, registry.toXML()) def add_fix(self, args, type, text, extra_args = {}): - registry = self.load_registry(args) + registry = self.load_one_registry(args) testcase = registry.getTest(args.testsuite, args.testcase) if testcase == None: - print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase) + logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase)) sys.exit(1) testcase.addFix(XITFix.createFromType(type, text, extra_args)) @@ -785,11 +996,11 @@ class XITTestRegistryCLI: self.add_fix(args, "rpm", args.rpm) def rm_fix(self, args, type, text): - registry = self.load_registry(args) + registry = self.load_one_registry(args) testcase = registry.getTest(args.testsuite, args.testcase) if testcase == None: - print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase) + logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase)) sys.exit(1) testcase.removeFix(XITFix.createFromType(type, text)) @@ -802,11 +1013,11 @@ class XITTestRegistryCLI: self.rm_fix(args, "rpm", args.rpm) def set_status(self, args): - registry = self.load_registry(args) + registry = self.load_one_registry(args) testcase = registry.getTest(args.testsuite, args.testcase) if testcase == None: - print >> sys.stderr, "Invalid test name '%s %s'" % (args.testsuite, args.testcase) + logging.error("Invalid test name '%s %s'" % (args.testsuite, args.testcase)) sys.exit(1) status = { "true" : True, @@ -817,13 +1028,13 @@ class XITTestRegistryCLI: try: testcase.status = status[args.status] except KeyError: - print >> sys.stderr, "Invalid status code, allowed are %s" % ",".join(status.keys()) + logging.error("Invalid status code, allowed are %s" % ",".join(status.keys())) sys.exit(1) self.registry_from_string(args, registry.toXML()) def set_date(self, args): - registry = self.load_registry(args) + registry = self.load_one_registry(args) date = args.date if date != None: date = time.strptime(date, "%Y-%m-%d") @@ -834,7 +1045,7 @@ class XITTestRegistryCLI: self.registry_from_string(args, registry.toXML()) def set_modversion(self, args): - registry = self.load_registry(args) + registry = self.load_one_registry(args) name = args.name version = args.version type = args.type if args.type else "git" @@ -851,14 +1062,17 @@ class XITTestRegistryCLI: "compare the test result with the registry of known test " "successes/failures.\n") parser.add_argument("-f", "--file", help="file containing XIT test registry, modified in-place (default: stdin/stdout) ", action="store", required=False) + parser.add_argument("-r", "--regname", metavar="registry-name", default=None, help="Work on the named test registry (defaults to first if not given) ", action="store", required=False) + parser.add_argument("--output-format", dest="printer", default=XITCLIPrinter(), required=False, + action=XITPrinterPicker, help="Pick output format (html, text ,default: text)") subparsers = parser.add_subparsers(title="Actions", help=None) list_subparser = subparsers.add_parser("list", help="List all test cases") list_subparser.set_defaults(func = self.list_tests) info_subparser = subparsers.add_parser("info", help="Print info about a specific test case") - info_subparser.add_argument("testsuite", nargs=1, default=None, help="Test Suite name") - info_subparser.add_argument("testcase", nargs=1, default=None, help="Test Case name") + info_subparser.add_argument("testsuite", default=None, help="Test Suite name") + info_subparser.add_argument("testcase", default=None, help="Test Case name") info_subparser.set_defaults(func = self.show_test_info) verify_subparser = subparsers.add_parser("verify", help="Compare JUnit test results against the registry") @@ -867,19 +1081,19 @@ class XITTestRegistryCLI: verify_subparser.set_defaults(func = self.verify_results) compare_subparser = subparsers.add_parser("compare", help="Compare two test registries") - compare_subparser.add_argument("reg1", metavar="registry1.xml", nargs=1, help="Registry file no 1") - compare_subparser.add_argument("reg2", metavar="registry2.xml", nargs=1, help="Registry file no 2") + compare_subparser.add_argument("reg1", metavar="registry1.xml", help="Registry file no 1") + compare_subparser.add_argument("reg2", metavar="registry2.xml", help="Registry file no 2") compare_subparser.set_defaults(func = self.compare_registries) - import_subparser = subparsers.add_parser("create", help="Create new XIT registry from JUnit test results") - import_subparser.add_argument("results", metavar="results.xml", help="The XML file containing test results") - import_subparser.add_argument("--name", nargs=1, help="Human-readable name for registry (default: the filename)") - import_subparser.add_argument("--auto-modversion", metavar="TYPE", nargs=1, help="Try to automatically get module versions for selected modules (default: rpm)") - import_subparser.set_defaults(func = self.create_registry) + create_subparser = subparsers.add_parser("create", help="Create new XIT registry from JUnit test results") + create_subparser.add_argument("results", metavar="results.xml", nargs="+", help="The XML file(s) containing test results") + create_subparser.add_argument("--name", nargs=1, help="Human-readable name for registry (default: the filename)") + create_subparser.add_argument("--auto-modversion", metavar="TYPE", nargs=1, help="Try to automatically get module versions for selected modules (default: rpm)") + create_subparser.set_defaults(func = self.create_registries) merge_subparser = subparsers.add_parser("merge", help="Merge two registries together") - merge_subparser.add_argument("reg1", metavar="registry1.xml", nargs=1, help="Registry file no 1") - merge_subparser.add_argument("reg2", metavar="registry2.xml", nargs=1, help="Registry file no 2") + merge_subparser.add_argument("reg1", metavar="registry1.xml", help="Registry file no 1") + merge_subparser.add_argument("reg2", metavar="registry2.xml", help="Registry file no 2") merge_subparser.add_argument("--add", default=True, action="store_true", help="Merge new test cases from registry 2 into registry 1, leaving existing test cases unmodified") merge_subparser.set_defaults(func = self.merge_registries) @@ -932,22 +1146,45 @@ class XITTestRegistryCLI: return parser - def load_registry(self, args): + def load_one_registry(self, args): + """Load and return a single registry from args.file. If args.name is + set, search for a named registry, otherwise return the first""" + if args.file == None: + logging.error("Reading from stdin") + args.file = sys.stdin + + registries = self.load_registry_from_file(args.file) + if args.regname != None: + for r in registries: + if r.name == args.regname: + return r + logging.error("Failed to find requested registry %s." % args.regname) + sys.exit(1) + else: + logging.warning("Multiple registries found, but no name given. Using first.") + return registries[0] + + + def load_registries(self, args): + """Load and return all registries from args.file. If args.name is + set, search for a named registry and return only that.""" if args.file == None: - print >> sys.stderr, "Reading from stdin" + logging.error("Reading from stdin") args.file = sys.stdin - return self.load_registry_from_file(args.file) + if args.regname: + return [self.load_one_registry(args)] + else: + return self.load_registry_from_file(args.file) def load_registry_from_file(self, path): + """Load and return a registry list from the given path.""" registries = XITTestRegistry.fromXML(path) - if len(registries) > 1: - print >> sys.stderr, "More than one registry found in input file, this is not supported yet. Using first one only." - elif len(registries) == 0: - print >> sys.stderr, "Failed to parse input file." + if len(registries) == 0: + logging.error("Failed to parse input file.") sys.exit(1) - return registries[0] + return registries def write_to_registry(self, f, msg): print >> f, msg |