diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2020-04-01 16:11:40 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2020-04-01 16:11:40 -0700 |
commit | 668f1e9267415153e30bea03828c0530874e92e4 (patch) | |
tree | d0333babe300648e72c6f202f53654a0651501dc /tools | |
parent | 397a97946798890b9bdaa6122fcfad7147690670 (diff) | |
parent | e23349af9ee25a5760112a2f8476b94a4ec86f1c (diff) |
Merge tag 'linux-kselftest-kunit-5.7-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest
Pull kunit updates from Shuah Khan:
"This kunit update consists of:
- debugfs support for displaying kunit test suite results.
This is especially useful for module-loaded tests to allow
disentangling of test result display from other dmesg events.
CONFIG_KUNIT_DEBUGFS enables/disables the debugfs support.
- Several fixes and improvements to kunit framework and tool"
* tag 'linux-kselftest-kunit-5.7-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest:
kunit: tool: add missing test data file content
kunit: update documentation to describe debugfs representation
kunit: subtests should be indented 4 spaces according to TAP
kunit: add log test
kunit: add debugfs /sys/kernel/debug/kunit/<suite>/results display
Documentation: kunit: Make the KUnit documentation less UML-specific
Fix linked-list KUnit test when run multiple times
kunit: kunit_tool: Allow .kunitconfig to disable config items
kunit: Always print actual pointer values in asserts
kunit: add --make_options
kunit: Run all KUnit tests through allyesconfig
kunit: kunit_parser: make parser more robust
Diffstat (limited to 'tools')
-rw-r--r-- | tools/testing/kunit/.gitattributes | 1 | ||||
-rw-r--r-- | tools/testing/kunit/configs/broken_on_uml.config | 41 | ||||
-rwxr-xr-x | tools/testing/kunit/kunit.py | 38 | ||||
-rw-r--r-- | tools/testing/kunit/kunit_config.py | 41 | ||||
-rw-r--r-- | tools/testing/kunit/kunit_kernel.py | 84 | ||||
-rw-r--r-- | tools/testing/kunit/kunit_parser.py | 51 | ||||
-rwxr-xr-x | tools/testing/kunit/kunit_tool_test.py | 108 | ||||
-rw-r--r-- | tools/testing/kunit/test_data/test_config_printk_time.log | 31 | ||||
-rw-r--r-- | tools/testing/kunit/test_data/test_interrupted_tap_output.log | 37 | ||||
-rw-r--r-- | tools/testing/kunit/test_data/test_kernel_panic_interrupt.log | 25 | ||||
-rw-r--r-- | tools/testing/kunit/test_data/test_multiple_prefixes.log | 31 | ||||
-rw-r--r-- | tools/testing/kunit/test_data/test_output_with_prefix_isolated_correctly.log | 33 | ||||
-rw-r--r-- | tools/testing/kunit/test_data/test_pound_no_prefix.log | 33 | ||||
-rw-r--r-- | tools/testing/kunit/test_data/test_pound_sign.log | 33 |
14 files changed, 496 insertions, 91 deletions
diff --git a/tools/testing/kunit/.gitattributes b/tools/testing/kunit/.gitattributes new file mode 100644 index 000000000000..5b7da1fc3b8f --- /dev/null +++ b/tools/testing/kunit/.gitattributes @@ -0,0 +1 @@ +test_data/* binary diff --git a/tools/testing/kunit/configs/broken_on_uml.config b/tools/testing/kunit/configs/broken_on_uml.config new file mode 100644 index 000000000000..239b9f03da2c --- /dev/null +++ b/tools/testing/kunit/configs/broken_on_uml.config @@ -0,0 +1,41 @@ +# These are currently broken on UML and prevent allyesconfig from building +# CONFIG_STATIC_LINK is not set +# CONFIG_UML_NET_VECTOR is not set +# CONFIG_UML_NET_VDE is not set +# CONFIG_UML_NET_PCAP is not set +# CONFIG_NET_PTP_CLASSIFY is not set +# CONFIG_IP_VS is not set +# CONFIG_BRIDGE_EBT_BROUTE is not set +# CONFIG_BRIDGE_EBT_T_FILTER is not set +# CONFIG_BRIDGE_EBT_T_NAT is not set +# CONFIG_MTD_NAND_CADENCE is not set +# CONFIG_MTD_NAND_NANDSIM is not set +# CONFIG_BLK_DEV_NULL_BLK is not set +# CONFIG_BLK_DEV_RAM is not set +# CONFIG_SCSI_DEBUG is not set +# CONFIG_NET_VENDOR_XILINX is not set +# CONFIG_NULL_TTY is not set +# CONFIG_PTP_1588_CLOCK is not set +# CONFIG_PINCTRL_EQUILIBRIUM is not set +# CONFIG_DMABUF_SELFTESTS is not set +# CONFIG_COMEDI is not set +# CONFIG_XIL_AXIS_FIFO is not set +# CONFIG_EXFAT_FS is not set +# CONFIG_STM_DUMMY is not set +# CONFIG_FSI_MASTER_ASPEED is not set +# CONFIG_JFS_FS is not set +# CONFIG_UBIFS_FS is not set +# CONFIG_CRAMFS is not set +# CONFIG_CRYPTO_DEV_SAFEXCEL is not set +# CONFIG_CRYPTO_DEV_AMLOGIC_GXL is not set +# CONFIG_KCOV is not set +# CONFIG_LKDTM is not set +# CONFIG_REED_SOLOMON_TEST is not set +# CONFIG_TEST_RHASHTABLE is not set +# CONFIG_TEST_MEMINIT is not set +# CONFIG_NETWORK_PHY_TIMESTAMPING is not set +# CONFIG_DEBUG_INFO_BTF is not set +# CONFIG_PTP_1588_CLOCK_INES is not set +# CONFIG_QCOM_CPR is not set +# CONFIG_RESET_BRCMSTB_RESCAL is not set +# CONFIG_RESET_INTEL_GW is not set diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 180ad1e1b04f..7dca74774dd2 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -22,7 +22,9 @@ import kunit_parser KunitResult = namedtuple('KunitResult', ['status','result']) -KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 'build_dir', 'defconfig']) +KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', + 'build_dir', 'defconfig', + 'alltests', 'make_options']) KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] @@ -47,7 +49,7 @@ def get_kernel_root_path(): def run_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitRequest) -> KunitResult: config_start = time.time() - success = linux.build_reconfig(request.build_dir) + success = linux.build_reconfig(request.build_dir, request.make_options) config_end = time.time() if not success: return KunitResult(KunitStatus.CONFIG_FAILURE, 'could not configure kernel') @@ -55,24 +57,24 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, kunit_parser.print_with_timestamp('Building KUnit Kernel ...') build_start = time.time() - success = linux.build_um_kernel(request.jobs, request.build_dir) + success = linux.build_um_kernel(request.alltests, + request.jobs, + request.build_dir, + request.make_options) build_end = time.time() if not success: return KunitResult(KunitStatus.BUILD_FAILURE, 'could not build kernel') kunit_parser.print_with_timestamp('Starting KUnit Kernel ...') test_start = time.time() - - test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, - [], - 'Tests not Parsed.') + kunit_output = linux.run_kernel( + timeout=None if request.alltests else request.timeout, + build_dir=request.build_dir) if request.raw_output: - kunit_parser.raw_output( - linux.run_kernel(timeout=request.timeout, - build_dir=request.build_dir)) + raw_output = kunit_parser.raw_output(kunit_output) + isolated = list(kunit_parser.isolate_kunit_output(raw_output)) + test_result = kunit_parser.parse_test_result(isolated) else: - kunit_output = linux.run_kernel(timeout=request.timeout, - build_dir=request.build_dir) test_result = kunit_parser.parse_run_tests(kunit_output) test_end = time.time() @@ -120,6 +122,14 @@ def main(argv, linux=None): help='Uses a default .kunitconfig.', action='store_true') + run_parser.add_argument('--alltests', + help='Run all KUnit tests through allyesconfig', + action='store_true') + + run_parser.add_argument('--make_options', + help='X=Y make option, can be repeated.', + action='append') + cli_args = parser.parse_args(argv) if cli_args.subcommand == 'run': @@ -143,7 +153,9 @@ def main(argv, linux=None): cli_args.timeout, cli_args.jobs, cli_args.build_dir, - cli_args.defconfig) + cli_args.defconfig, + cli_args.alltests, + cli_args.make_options) result = run_tests(linux, request) if result.status != KunitStatus.SUCCESS: sys.exit(1) diff --git a/tools/testing/kunit/kunit_config.py b/tools/testing/kunit/kunit_config.py index ebf3942b23f5..e75063d603b5 100644 --- a/tools/testing/kunit/kunit_config.py +++ b/tools/testing/kunit/kunit_config.py @@ -9,16 +9,18 @@ import collections import re -CONFIG_IS_NOT_SET_PATTERN = r'^# CONFIG_\w+ is not set$' -CONFIG_PATTERN = r'^CONFIG_\w+=\S+$' - -KconfigEntryBase = collections.namedtuple('KconfigEntry', ['raw_entry']) +CONFIG_IS_NOT_SET_PATTERN = r'^# CONFIG_(\w+) is not set$' +CONFIG_PATTERN = r'^CONFIG_(\w+)=(\S+)$' +KconfigEntryBase = collections.namedtuple('KconfigEntry', ['name', 'value']) class KconfigEntry(KconfigEntryBase): def __str__(self) -> str: - return self.raw_entry + if self.value == 'n': + return r'# CONFIG_%s is not set' % (self.name) + else: + return r'CONFIG_%s=%s' % (self.name, self.value) class KconfigParseError(Exception): @@ -38,7 +40,17 @@ class Kconfig(object): self._entries.append(entry) def is_subset_of(self, other: 'Kconfig') -> bool: - return self.entries().issubset(other.entries()) + for a in self.entries(): + found = False + for b in other.entries(): + if a.name != b.name: + continue + if a.value != b.value: + return False + found = True + if a.value != 'n' and found == False: + return False + return True def write_to_file(self, path: str) -> None: with open(path, 'w') as f: @@ -54,9 +66,20 @@ class Kconfig(object): line = line.strip() if not line: continue - elif config_matcher.match(line) or is_not_set_matcher.match(line): - self._entries.append(KconfigEntry(line)) - elif line[0] == '#': + + match = config_matcher.match(line) + if match: + entry = KconfigEntry(match.group(1), match.group(2)) + self.add_entry(entry) + continue + + empty_match = is_not_set_matcher.match(line) + if empty_match: + entry = KconfigEntry(empty_match.group(1), 'n') + self.add_entry(entry) + continue + + if line[0] == '#': continue else: raise KconfigParseError('Failed to parse: ' + line) diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index d99ae75ef72f..63dbda2d029f 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -10,11 +10,16 @@ import logging import subprocess import os +import signal + +from contextlib import ExitStack import kunit_config +import kunit_parser KCONFIG_PATH = '.config' kunitconfig_path = '.kunitconfig' +BROKEN_ALLCONFIG_PATH = 'tools/testing/kunit/configs/broken_on_uml.config' class ConfigError(Exception): """Represents an error trying to configure the Linux kernel.""" @@ -35,19 +40,40 @@ class LinuxSourceTreeOperations(object): except subprocess.CalledProcessError as e: raise ConfigError(e.output) - def make_olddefconfig(self, build_dir): + def make_olddefconfig(self, build_dir, make_options): command = ['make', 'ARCH=um', 'olddefconfig'] + if make_options: + command.extend(make_options) if build_dir: command += ['O=' + build_dir] try: - subprocess.check_output(command) + subprocess.check_output(command, stderr=subprocess.PIPE) except OSError as e: raise ConfigError('Could not call make command: ' + e) except subprocess.CalledProcessError as e: raise ConfigError(e.output) - def make(self, jobs, build_dir): + def make_allyesconfig(self): + kunit_parser.print_with_timestamp( + 'Enabling all CONFIGs for UML...') + process = subprocess.Popen( + ['make', 'ARCH=um', 'allyesconfig'], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT) + process.wait() + kunit_parser.print_with_timestamp( + 'Disabling broken configs to run KUnit tests...') + with ExitStack() as es: + config = open(KCONFIG_PATH, 'a') + disable = open(BROKEN_ALLCONFIG_PATH, 'r').read() + config.write(disable) + kunit_parser.print_with_timestamp( + 'Starting Kernel with all configs takes a few minutes...') + + def make(self, jobs, build_dir, make_options): command = ['make', 'ARCH=um', '--jobs=' + str(jobs)] + if make_options: + command.extend(make_options) if build_dir: command += ['O=' + build_dir] try: @@ -57,18 +83,16 @@ class LinuxSourceTreeOperations(object): except subprocess.CalledProcessError as e: raise BuildError(e.output) - def linux_bin(self, params, timeout, build_dir): + def linux_bin(self, params, timeout, build_dir, outfile): """Runs the Linux UML binary. Must be named 'linux'.""" linux_bin = './linux' if build_dir: linux_bin = os.path.join(build_dir, 'linux') - process = subprocess.Popen( - [linux_bin] + params, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - process.wait(timeout=timeout) - return process + with open(outfile, 'w') as output: + process = subprocess.Popen([linux_bin] + params, + stdout=output, + stderr=subprocess.STDOUT) + process.wait(timeout) def get_kconfig_path(build_dir): @@ -84,6 +108,7 @@ class LinuxSourceTree(object): self._kconfig = kunit_config.Kconfig() self._kconfig.read_from_file(kunitconfig_path) self._ops = LinuxSourceTreeOperations() + signal.signal(signal.SIGINT, self.signal_handler) def clean(self): try: @@ -107,19 +132,19 @@ class LinuxSourceTree(object): return False return True - def build_config(self, build_dir): + def build_config(self, build_dir, make_options): kconfig_path = get_kconfig_path(build_dir) if build_dir and not os.path.exists(build_dir): os.mkdir(build_dir) self._kconfig.write_to_file(kconfig_path) try: - self._ops.make_olddefconfig(build_dir) + self._ops.make_olddefconfig(build_dir, make_options) except ConfigError as e: logging.error(e) return False return self.validate_config(build_dir) - def build_reconfig(self, build_dir): + def build_reconfig(self, build_dir, make_options): """Creates a new .config if it is not a subset of the .kunitconfig.""" kconfig_path = get_kconfig_path(build_dir) if os.path.exists(kconfig_path): @@ -128,26 +153,33 @@ class LinuxSourceTree(object): if not self._kconfig.is_subset_of(existing_kconfig): print('Regenerating .config ...') os.remove(kconfig_path) - return self.build_config(build_dir) + return self.build_config(build_dir, make_options) else: return True else: print('Generating .config ...') - return self.build_config(build_dir) + return self.build_config(build_dir, make_options) - def build_um_kernel(self, jobs, build_dir): + def build_um_kernel(self, alltests, jobs, build_dir, make_options): + if alltests: + self._ops.make_allyesconfig() try: - self._ops.make_olddefconfig(build_dir) - self._ops.make(jobs, build_dir) + self._ops.make_olddefconfig(build_dir, make_options) + self._ops.make(jobs, build_dir, make_options) except (ConfigError, BuildError) as e: logging.error(e) return False return self.validate_config(build_dir) - def run_kernel(self, args=[], timeout=None, build_dir=''): - args.extend(['mem=256M']) - process = self._ops.linux_bin(args, timeout, build_dir) - with open(os.path.join(build_dir, 'test.log'), 'w') as f: - for line in process.stdout: - f.write(line.rstrip().decode('ascii') + '\n') - yield line.rstrip().decode('ascii') + def run_kernel(self, args=[], build_dir='', timeout=None): + args.extend(['mem=1G']) + outfile = 'test.log' + self._ops.linux_bin(args, timeout, build_dir, outfile) + subprocess.call(['stty', 'sane']) + with open(outfile, 'r') as file: + for line in file: + yield line + + def signal_handler(self, sig, frame): + logging.error('Build interruption occurred. Cleaning console.') + subprocess.call(['stty', 'sane']) diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 4ffbae0f6732..64aac9dcd431 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -46,23 +46,26 @@ class TestStatus(Enum): TEST_CRASHED = auto() NO_TESTS = auto() -kunit_start_re = re.compile(r'^TAP version [0-9]+$') -kunit_end_re = re.compile('List of all partitions:') +kunit_start_re = re.compile(r'TAP version [0-9]+$') +kunit_end_re = re.compile('(List of all partitions:|' + 'Kernel panic - not syncing: VFS:|reboot: System halted)') def isolate_kunit_output(kernel_output): started = False for line in kernel_output: - if kunit_start_re.match(line): + if kunit_start_re.search(line): + prefix_len = len(line.split('TAP version')[0]) started = True - yield line - elif kunit_end_re.match(line): + yield line[prefix_len:] if prefix_len > 0 else line + elif kunit_end_re.search(line): break elif started: - yield line + yield line[prefix_len:] if prefix_len > 0 else line def raw_output(kernel_output): for line in kernel_output: print(line) + yield line DIVIDER = '=' * 60 @@ -91,7 +94,7 @@ def print_log(log): for m in log: print_with_timestamp(m) -TAP_ENTRIES = re.compile(r'^(TAP|\t?ok|\t?not ok|\t?[0-9]+\.\.[0-9]+|\t?#).*$') +TAP_ENTRIES = re.compile(r'^(TAP|[\s]*ok|[\s]*not ok|[\s]*[0-9]+\.\.[0-9]+|[\s]*#).*$') def consume_non_diagnositic(lines: List[str]) -> None: while lines and not TAP_ENTRIES.match(lines[0]): @@ -104,22 +107,20 @@ def save_non_diagnositic(lines: List[str], test_case: TestCase) -> None: OkNotOkResult = namedtuple('OkNotOkResult', ['is_ok','description', 'text']) -OK_NOT_OK_SUBTEST = re.compile(r'^\t(ok|not ok) [0-9]+ - (.*)$') +OK_NOT_OK_SUBTEST = re.compile(r'^[\s]+(ok|not ok) [0-9]+ - (.*)$') OK_NOT_OK_MODULE = re.compile(r'^(ok|not ok) [0-9]+ - (.*)$') -def parse_ok_not_ok_test_case(lines: List[str], - test_case: TestCase, - expecting_test_case: bool) -> bool: +def parse_ok_not_ok_test_case(lines: List[str], test_case: TestCase) -> bool: save_non_diagnositic(lines, test_case) if not lines: - if expecting_test_case: - test_case.status = TestStatus.TEST_CRASHED - return True - else: - return False + test_case.status = TestStatus.TEST_CRASHED + return True line = lines[0] match = OK_NOT_OK_SUBTEST.match(line) + while not match and lines: + line = lines.pop(0) + match = OK_NOT_OK_SUBTEST.match(line) if match: test_case.log.append(lines.pop(0)) test_case.name = match.group(2) @@ -133,7 +134,7 @@ def parse_ok_not_ok_test_case(lines: List[str], else: return False -SUBTEST_DIAGNOSTIC = re.compile(r'^\t# .*?: (.*)$') +SUBTEST_DIAGNOSTIC = re.compile(r'^[\s]+# .*?: (.*)$') DIAGNOSTIC_CRASH_MESSAGE = 'kunit test case crashed!' def parse_diagnostic(lines: List[str], test_case: TestCase) -> bool: @@ -150,17 +151,17 @@ def parse_diagnostic(lines: List[str], test_case: TestCase) -> bool: else: return False -def parse_test_case(lines: List[str], expecting_test_case: bool) -> TestCase: +def parse_test_case(lines: List[str]) -> TestCase: test_case = TestCase() save_non_diagnositic(lines, test_case) while parse_diagnostic(lines, test_case): pass - if parse_ok_not_ok_test_case(lines, test_case, expecting_test_case): + if parse_ok_not_ok_test_case(lines, test_case): return test_case else: return None -SUBTEST_HEADER = re.compile(r'^\t# Subtest: (.*)$') +SUBTEST_HEADER = re.compile(r'^[\s]+# Subtest: (.*)$') def parse_subtest_header(lines: List[str]) -> str: consume_non_diagnositic(lines) @@ -173,7 +174,7 @@ def parse_subtest_header(lines: List[str]) -> str: else: return None -SUBTEST_PLAN = re.compile(r'\t[0-9]+\.\.([0-9]+)') +SUBTEST_PLAN = re.compile(r'[\s]+[0-9]+\.\.([0-9]+)') def parse_subtest_plan(lines: List[str]) -> int: consume_non_diagnositic(lines) @@ -234,11 +235,11 @@ def parse_test_suite(lines: List[str]) -> TestSuite: expected_test_case_num = parse_subtest_plan(lines) if not expected_test_case_num: return None - test_case = parse_test_case(lines, expected_test_case_num > 0) - expected_test_case_num -= 1 - while test_case: + while expected_test_case_num > 0: + test_case = parse_test_case(lines) + if not test_case: + break test_suite.cases.append(test_case) - test_case = parse_test_case(lines, expected_test_case_num > 0) expected_test_case_num -= 1 if parse_ok_not_ok_test_suite(lines, test_suite): test_suite.status = bubble_up_test_case_errors(test_suite) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index cba97756ac4a..984588d6ba95 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -37,7 +37,7 @@ class KconfigTest(unittest.TestCase): self.assertTrue(kconfig0.is_subset_of(kconfig0)) kconfig1 = kunit_config.Kconfig() - kconfig1.add_entry(kunit_config.KconfigEntry('CONFIG_TEST=y')) + kconfig1.add_entry(kunit_config.KconfigEntry('TEST', 'y')) self.assertTrue(kconfig1.is_subset_of(kconfig1)) self.assertTrue(kconfig0.is_subset_of(kconfig1)) self.assertFalse(kconfig1.is_subset_of(kconfig0)) @@ -51,15 +51,15 @@ class KconfigTest(unittest.TestCase): expected_kconfig = kunit_config.Kconfig() expected_kconfig.add_entry( - kunit_config.KconfigEntry('CONFIG_UML=y')) + kunit_config.KconfigEntry('UML', 'y')) expected_kconfig.add_entry( - kunit_config.KconfigEntry('CONFIG_MMU=y')) + kunit_config.KconfigEntry('MMU', 'y')) expected_kconfig.add_entry( - kunit_config.KconfigEntry('CONFIG_TEST=y')) + kunit_config.KconfigEntry('TEST', 'y')) expected_kconfig.add_entry( - kunit_config.KconfigEntry('CONFIG_EXAMPLE_TEST=y')) + kunit_config.KconfigEntry('EXAMPLE_TEST', 'y')) expected_kconfig.add_entry( - kunit_config.KconfigEntry('# CONFIG_MK8 is not set')) + kunit_config.KconfigEntry('MK8', 'n')) self.assertEqual(kconfig.entries(), expected_kconfig.entries()) @@ -68,15 +68,15 @@ class KconfigTest(unittest.TestCase): expected_kconfig = kunit_config.Kconfig() expected_kconfig.add_entry( - kunit_config.KconfigEntry('CONFIG_UML=y')) + kunit_config.KconfigEntry('UML', 'y')) expected_kconfig.add_entry( - kunit_config.KconfigEntry('CONFIG_MMU=y')) + kunit_config.KconfigEntry('MMU', 'y')) expected_kconfig.add_entry( - kunit_config.KconfigEntry('CONFIG_TEST=y')) + kunit_config.KconfigEntry('TEST', 'y')) expected_kconfig.add_entry( - kunit_config.KconfigEntry('CONFIG_EXAMPLE_TEST=y')) + kunit_config.KconfigEntry('EXAMPLE_TEST', 'y')) expected_kconfig.add_entry( - kunit_config.KconfigEntry('# CONFIG_MK8 is not set')) + kunit_config.KconfigEntry('MK8', 'n')) expected_kconfig.write_to_file(kconfig_path) @@ -108,6 +108,36 @@ class KUnitParserTest(unittest.TestCase): self.assertContains('ok 1 - example', result) file.close() + def test_output_with_prefix_isolated_correctly(self): + log_path = get_absolute_path( + 'test_data/test_pound_sign.log') + with open(log_path) as file: + result = kunit_parser.isolate_kunit_output(file.readlines()) + self.assertContains('TAP version 14\n', result) + self.assertContains(' # Subtest: kunit-resource-test', result) + self.assertContains(' 1..5', result) + self.assertContains(' ok 1 - kunit_resource_test_init_resources', result) + self.assertContains(' ok 2 - kunit_resource_test_alloc_resource', result) + self.assertContains(' ok 3 - kunit_resource_test_destroy_resource', result) + self.assertContains(' foo bar #', result) + self.assertContains(' ok 4 - kunit_resource_test_cleanup_resources', result) + self.assertContains(' ok 5 - kunit_resource_test_proper_free_ordering', result) + self.assertContains('ok 1 - kunit-resource-test', result) + self.assertContains(' foo bar # non-kunit output', result) + self.assertContains(' # Subtest: kunit-try-catch-test', result) + self.assertContains(' 1..2', result) + self.assertContains(' ok 1 - kunit_test_try_catch_successful_try_no_catch', + result) + self.assertContains(' ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch', + result) + self.assertContains('ok 2 - kunit-try-catch-test', result) + self.assertContains(' # Subtest: string-stream-test', result) + self.assertContains(' 1..3', result) + self.assertContains(' ok 1 - string_stream_test_empty_on_creation', result) + self.assertContains(' ok 2 - string_stream_test_not_empty_after_add', result) + self.assertContains(' ok 3 - string_stream_test_get_string', result) + self.assertContains('ok 3 - string-stream-test', result) + def test_parse_successful_test_log(self): all_passed_log = get_absolute_path( 'test_data/test_is_test_passed-all_passed.log') @@ -150,6 +180,45 @@ class KUnitParserTest(unittest.TestCase): result.status) file.close() + def test_ignores_prefix_printk_time(self): + prefix_log = get_absolute_path( + 'test_data/test_config_printk_time.log') + with open(prefix_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_ignores_multiple_prefixes(self): + prefix_log = get_absolute_path( + 'test_data/test_multiple_prefixes.log') + with open(prefix_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_prefix_mixed_kernel_output(self): + mixed_prefix_log = get_absolute_path( + 'test_data/test_interrupted_tap_output.log') + with open(mixed_prefix_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_prefix_poundsign(self): + pound_log = get_absolute_path('test_data/test_pound_sign.log') + with open(pound_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_kernel_panic_end(self): + panic_log = get_absolute_path('test_data/test_kernel_panic_interrupt.log') + with open(panic_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + + def test_pound_no_prefix(self): + pound_log = get_absolute_path('test_data/test_pound_no_prefix.log') + with open(pound_log) as file: + result = kunit_parser.parse_run_tests(file.readlines()) + self.assertEqual('kunit-resource-test', result.suites[0].name) + class StrContains(str): def __eq__(self, other): return self in other @@ -174,7 +243,8 @@ class KUnitMainTest(unittest.TestCase): kunit.main(['run'], self.linux_source_mock) assert self.linux_source_mock.build_reconfig.call_count == 1 assert self.linux_source_mock.run_kernel.call_count == 1 - self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='', timeout=300) + self.linux_source_mock.run_kernel.assert_called_once_with( + build_dir='', timeout=300) self.print_mock.assert_any_call(StrContains('Testing complete.')) def test_run_passes_args_fail(self): @@ -189,25 +259,27 @@ class KUnitMainTest(unittest.TestCase): def test_run_raw_output(self): self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) - kunit.main(['run', '--raw_output'], self.linux_source_mock) + with self.assertRaises(SystemExit) as e: + kunit.main(['run', '--raw_output'], self.linux_source_mock) + assert type(e.exception) == SystemExit + assert e.exception.code == 1 assert self.linux_source_mock.build_reconfig.call_count == 1 assert self.linux_source_mock.run_kernel.call_count == 1 - for kall in self.print_mock.call_args_list: - assert kall != mock.call(StrContains('Testing complete.')) - assert kall != mock.call(StrContains(' 0 tests run')) def test_run_timeout(self): timeout = 3453 kunit.main(['run', '--timeout', str(timeout)], self.linux_source_mock) assert self.linux_source_mock.build_reconfig.call_count == 1 - self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='', timeout=timeout) + self.linux_source_mock.run_kernel.assert_called_once_with( + build_dir='', timeout=timeout) self.print_mock.assert_any_call(StrContains('Testing complete.')) def test_run_builddir(self): build_dir = '.kunit' kunit.main(['run', '--build_dir', build_dir], self.linux_source_mock) assert self.linux_source_mock.build_reconfig.call_count == 1 - self.linux_source_mock.run_kernel.assert_called_once_with(build_dir=build_dir, timeout=300) + self.linux_source_mock.run_kernel.assert_called_once_with( + build_dir=build_dir, timeout=300) self.print_mock.assert_any_call(StrContains('Testing complete.')) if __name__ == '__main__': diff --git a/tools/testing/kunit/test_data/test_config_printk_time.log b/tools/testing/kunit/test_data/test_config_printk_time.log new file mode 100644 index 000000000000..c02ca773946d --- /dev/null +++ b/tools/testing/kunit/test_data/test_config_printk_time.log @@ -0,0 +1,31 @@ +[ 0.060000] printk: console [mc-1] enabled +[ 0.060000] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000] TAP version 14 +[ 0.060000] # Subtest: kunit-resource-test +[ 0.060000] 1..5 +[ 0.060000] ok 1 - kunit_resource_test_init_resources +[ 0.060000] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000] ok 1 - kunit-resource-test +[ 0.060000] # Subtest: kunit-try-catch-test +[ 0.060000] 1..2 +[ 0.060000] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000] ok 2 - kunit-try-catch-test +[ 0.060000] # Subtest: string-stream-test +[ 0.060000] 1..3 +[ 0.060000] ok 1 - string_stream_test_empty_on_creation +[ 0.060000] ok 2 - string_stream_test_not_empty_after_add +[ 0.060000] ok 3 - string_stream_test_get_string +[ 0.060000] ok 3 - string-stream-test +[ 0.060000] List of all partitions: +[ 0.060000] No filesystem could mount root, tried: +[ 0.060000] +[ 0.060000] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000] Stack: +[ 0.060000] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000] 6005fcec 601bc260 705c0000 3000000010
\ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_interrupted_tap_output.log b/tools/testing/kunit/test_data/test_interrupted_tap_output.log new file mode 100644 index 000000000000..5c73fb3a1c6f --- /dev/null +++ b/tools/testing/kunit/test_data/test_interrupted_tap_output.log @@ -0,0 +1,37 @@ +[ 0.060000] printk: console [mc-1] enabled +[ 0.060000] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000] TAP version 14 +[ 0.060000] # Subtest: kunit-resource-test +[ 0.060000] 1..5 +[ 0.060000] ok 1 - kunit_resource_test_init_resources +[ 0.060000] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000] kAFS: Red Hat AFS client v0.1 registering. +[ 0.060000] FS-Cache: Netfs 'afs' registered for caching +[ 0.060000] *** VALIDATE kAFS *** +[ 0.060000] Btrfs loaded, crc32c=crc32c-generic, debug=on, assert=on, integrity-checker=on, ref-verify=on +[ 0.060000] BTRFS: selftest: sectorsize: 4096 nodesize: 4096 +[ 0.060000] BTRFS: selftest: running btrfs free space cache tests +[ 0.060000] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000] ok 1 - kunit-resource-test +[ 0.060000] # Subtest: kunit-try-catch-test +[ 0.060000] 1..2 +[ 0.060000] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000] ok 2 - kunit-try-catch-test +[ 0.060000] # Subtest: string-stream-test +[ 0.060000] 1..3 +[ 0.060000] ok 1 - string_stream_test_empty_on_creation +[ 0.060000] ok 2 - string_stream_test_not_empty_after_add +[ 0.060000] ok 3 - string_stream_test_get_string +[ 0.060000] ok 3 - string-stream-test +[ 0.060000] List of all partitions: +[ 0.060000] No filesystem could mount root, tried: +[ 0.060000] +[ 0.060000] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000] Stack: +[ 0.060000] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000] 6005fcec 601bc260 705c0000 3000000010
\ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_kernel_panic_interrupt.log b/tools/testing/kunit/test_data/test_kernel_panic_interrupt.log new file mode 100644 index 000000000000..c045eee75f27 --- /dev/null +++ b/tools/testing/kunit/test_data/test_kernel_panic_interrupt.log @@ -0,0 +1,25 @@ +[ 0.060000] printk: console [mc-1] enabled +[ 0.060000] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000] TAP version 14 +[ 0.060000] # Subtest: kunit-resource-test +[ 0.060000] 1..5 +[ 0.060000] ok 1 - kunit_resource_test_init_resources +[ 0.060000] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000] ok 1 - kunit-resource-test +[ 0.060000] # Subtest: kunit-try-catch-test +[ 0.060000] 1..2 +[ 0.060000] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000] ok 2 - kunit-try-catch-test +[ 0.060000] # Subtest: string-stream-test +[ 0.060000] 1..3 +[ 0.060000] ok 1 - string_stream_test_empty_on_creation +[ 0.060000] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000] Stack: +[ 0.060000] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000] 6005fcec 601bc260 705c0000 3000000010
\ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_multiple_prefixes.log b/tools/testing/kunit/test_data/test_multiple_prefixes.log new file mode 100644 index 000000000000..bc48407dcc36 --- /dev/null +++ b/tools/testing/kunit/test_data/test_multiple_prefixes.log @@ -0,0 +1,31 @@ +[ 0.060000][ T1] printk: console [mc-1] enabled +[ 0.060000][ T1] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000][ T1] TAP version 14 +[ 0.060000][ T1] # Subtest: kunit-resource-test +[ 0.060000][ T1] 1..5 +[ 0.060000][ T1] ok 1 - kunit_resource_test_init_resources +[ 0.060000][ T1] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000][ T1] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000][ T1] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000][ T1] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000][ T1] ok 1 - kunit-resource-test +[ 0.060000][ T1] # Subtest: kunit-try-catch-test +[ 0.060000][ T1] 1..2 +[ 0.060000][ T1] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000][ T1] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000][ T1] ok 2 - kunit-try-catch-test +[ 0.060000][ T1] # Subtest: string-stream-test +[ 0.060000][ T1] 1..3 +[ 0.060000][ T1] ok 1 - string_stream_test_empty_on_creation +[ 0.060000][ T1] ok 2 - string_stream_test_not_empty_after_add +[ 0.060000][ T1] ok 3 - string_stream_test_get_string +[ 0.060000][ T1] ok 3 - string-stream-test +[ 0.060000][ T1] List of all partitions: +[ 0.060000][ T1] No filesystem could mount root, tried: +[ 0.060000][ T1] +[ 0.060000][ T1] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000][ T1] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000][ T1] Stack: +[ 0.060000][ T1] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000][ T1] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000][ T1] 6005fcec 601bc260 705c0000 3000000010
\ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_output_with_prefix_isolated_correctly.log b/tools/testing/kunit/test_data/test_output_with_prefix_isolated_correctly.log new file mode 100644 index 000000000000..0f87cdabebb0 --- /dev/null +++ b/tools/testing/kunit/test_data/test_output_with_prefix_isolated_correctly.log @@ -0,0 +1,33 @@ +[ 0.060000] printk: console [mc-1] enabled +[ 0.060000] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000] TAP version 14 +[ 0.060000] # Subtest: kunit-resource-test +[ 0.060000] 1..5 +[ 0.060000] ok 1 - kunit_resource_test_init_resources +[ 0.060000] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000] foo bar # +[ 0.060000] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000] ok 1 - kunit-resource-test +[ 0.060000] foo bar # non-kunit output +[ 0.060000] # Subtest: kunit-try-catch-test +[ 0.060000] 1..2 +[ 0.060000] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000] ok 2 - kunit-try-catch-test +[ 0.060000] # Subtest: string-stream-test +[ 0.060000] 1..3 +[ 0.060000] ok 1 - string_stream_test_empty_on_creation +[ 0.060000] ok 2 - string_stream_test_not_empty_after_add +[ 0.060000] ok 3 - string_stream_test_get_string +[ 0.060000] ok 3 - string-stream-test +[ 0.060000] List of all partitions: +[ 0.060000] No filesystem could mount root, tried: +[ 0.060000] +[ 0.060000] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000] Stack: +[ 0.060000] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000] 6005fcec 601bc260 705c0000 3000000010
\ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_pound_no_prefix.log b/tools/testing/kunit/test_data/test_pound_no_prefix.log new file mode 100644 index 000000000000..2ceb360be7d5 --- /dev/null +++ b/tools/testing/kunit/test_data/test_pound_no_prefix.log @@ -0,0 +1,33 @@ + printk: console [mc-1] enabled + random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 + TAP version 14 + # Subtest: kunit-resource-test + 1..5 + ok 1 - kunit_resource_test_init_resources + ok 2 - kunit_resource_test_alloc_resource + ok 3 - kunit_resource_test_destroy_resource + foo bar # + ok 4 - kunit_resource_test_cleanup_resources + ok 5 - kunit_resource_test_proper_free_ordering + ok 1 - kunit-resource-test + foo bar # non-kunit output + # Subtest: kunit-try-catch-test + 1..2 + ok 1 - kunit_test_try_catch_successful_try_no_catch + ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch + ok 2 - kunit-try-catch-test + # Subtest: string-stream-test + 1..3 + ok 1 - string_stream_test_empty_on_creation + ok 2 - string_stream_test_not_empty_after_add + ok 3 - string_stream_test_get_string + ok 3 - string-stream-test + List of all partitions: + No filesystem could mount root, tried: + + Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) + CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 + Stack: + 602086f8 601bc260 705c0000 705c0000 + 602086f8 6005fcec 705c0000 6002c6ab + 6005fcec 601bc260 705c0000 3000000010
\ No newline at end of file diff --git a/tools/testing/kunit/test_data/test_pound_sign.log b/tools/testing/kunit/test_data/test_pound_sign.log new file mode 100644 index 000000000000..28ffa5ba03bf --- /dev/null +++ b/tools/testing/kunit/test_data/test_pound_sign.log @@ -0,0 +1,33 @@ +[ 0.060000] printk: console [mc-1] enabled +[ 0.060000] random: get_random_bytes called from init_oops_id+0x35/0x40 with crng_init=0 +[ 0.060000] TAP version 14 +[ 0.060000] # Subtest: kunit-resource-test +[ 0.060000] 1..5 +[ 0.060000] ok 1 - kunit_resource_test_init_resources +[ 0.060000] ok 2 - kunit_resource_test_alloc_resource +[ 0.060000] ok 3 - kunit_resource_test_destroy_resource +[ 0.060000] foo bar # +[ 0.060000] ok 4 - kunit_resource_test_cleanup_resources +[ 0.060000] ok 5 - kunit_resource_test_proper_free_ordering +[ 0.060000] ok 1 - kunit-resource-test +[ 0.060000] foo bar # non-kunit output +[ 0.060000] # Subtest: kunit-try-catch-test +[ 0.060000] 1..2 +[ 0.060000] ok 1 - kunit_test_try_catch_successful_try_no_catch +[ 0.060000] ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch +[ 0.060000] ok 2 - kunit-try-catch-test +[ 0.060000] # Subtest: string-stream-test +[ 0.060000] 1..3 +[ 0.060000] ok 1 - string_stream_test_empty_on_creation +[ 0.060000] ok 2 - string_stream_test_not_empty_after_add +[ 0.060000] ok 3 - string_stream_test_get_string +[ 0.060000] ok 3 - string-stream-test +[ 0.060000] List of all partitions: +[ 0.060000] No filesystem could mount root, tried: +[ 0.060000] +[ 0.060000] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(98,0) +[ 0.060000] CPU: 0 PID: 1 Comm: swapper Not tainted 5.4.0-rc1-gea2dd7c0875e-dirty #2 +[ 0.060000] Stack: +[ 0.060000] 602086f8 601bc260 705c0000 705c0000 +[ 0.060000] 602086f8 6005fcec 705c0000 6002c6ab +[ 0.060000] 6005fcec 601bc260 705c0000 3000000010 |