Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 1 | """TOSA verification runner script.""" |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 2 | # Copyright (c) 2020-2023, ARM Limited. |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | import argparse |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 5 | import importlib |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 6 | import json |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 7 | import os |
| 8 | import queue |
| 9 | import threading |
| 10 | import traceback |
| 11 | from datetime import datetime |
| 12 | from pathlib import Path |
| 13 | |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 14 | import conformance.model_files as cmf |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 15 | import runner.tosa_test_presets as ttp |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 16 | from generator.datagenerator import GenerateError |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 17 | from runner.tosa_test_runner import TosaTestInvalid |
| 18 | from runner.tosa_test_runner import TosaTestRunner |
| 19 | from xunit import xunit |
| 20 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 21 | |
| 22 | def parseArgs(argv): |
| 23 | """Parse the arguments and return the settings.""" |
| 24 | parser = argparse.ArgumentParser() |
| 25 | group = parser.add_mutually_exclusive_group(required=True) |
| 26 | group.add_argument( |
| 27 | "-t", |
| 28 | "--test", |
| 29 | dest="test", |
| 30 | type=str, |
| 31 | nargs="+", |
| 32 | help="Test(s) to run", |
| 33 | ) |
| 34 | group.add_argument( |
| 35 | "-T", |
| 36 | "--test-list", |
| 37 | dest="test_list_file", |
| 38 | type=Path, |
| 39 | help="File containing list of tests to run (one per line)", |
| 40 | ) |
| 41 | parser.add_argument( |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 42 | "-r", |
| 43 | "--recursive", |
| 44 | dest="recursive_tests", |
| 45 | action="store_true", |
| 46 | help="Recursively search for tests", |
| 47 | ) |
| 48 | parser.add_argument( |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 49 | "--ref-model-path", |
| 50 | dest="ref_model_path", |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 51 | type=Path, |
| 52 | help="Path to TOSA reference model executable", |
| 53 | ) |
| 54 | parser.add_argument( |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 55 | "--generate-lib-path", |
| 56 | dest="generate_lib_path", |
| 57 | type=Path, |
| 58 | help=( |
| 59 | "Path to TOSA generate library. Defaults to " |
| 60 | "the library in the directory of `ref-model-path`" |
| 61 | ), |
| 62 | ) |
| 63 | parser.add_argument( |
Jeremy Johnson | 9c2fe6e | 2023-10-04 16:55:04 +0100 | [diff] [blame] | 64 | "--verify-lib-path", |
| 65 | dest="verify_lib_path", |
| 66 | type=Path, |
| 67 | help=( |
| 68 | "Path to TOSA verify library. Defaults to " |
| 69 | "the library in the directory of `ref-model-path`" |
| 70 | ), |
| 71 | ) |
| 72 | parser.add_argument( |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 73 | "--operator-fbs", |
| 74 | "--schema-path", |
| 75 | dest="schema_path", |
| 76 | type=Path, |
| 77 | help=( |
| 78 | "Path to TOSA reference model flat buffer schema. Defaults to " |
| 79 | f"`{cmf.DEFAULT_REF_MODEL_SCHEMA_PATH}` in parents parent directory of `ref-model-path`" |
| 80 | ), |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 81 | ) |
| 82 | parser.add_argument( |
| 83 | "--flatc-path", |
| 84 | dest="flatc_path", |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 85 | type=Path, |
| 86 | help=( |
| 87 | "Path to flatc executable. Defaults to " |
| 88 | f"`{cmf.DEFAULT_REF_MODEL_BUILD_FLATC_PATH}` in parent directory of `ref-model-path`" |
| 89 | ), |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 90 | ) |
| 91 | parser.add_argument( |
| 92 | "--ref-debug", |
| 93 | dest="ref_debug", |
| 94 | default="", |
| 95 | type=str, |
| 96 | help="Reference debug flag (low, med, high)", |
| 97 | ) |
| 98 | parser.add_argument( |
| 99 | "--ref-intermediates", |
| 100 | dest="ref_intermediates", |
| 101 | default=0, |
| 102 | type=int, |
| 103 | help="Reference model dumps intermediate tensors", |
| 104 | ) |
| 105 | parser.add_argument( |
| 106 | "-b", |
| 107 | "--binary", |
| 108 | dest="binary", |
| 109 | action="store_true", |
| 110 | help="Convert to using binary flatbuffers instead of JSON", |
| 111 | ) |
| 112 | parser.add_argument( |
| 113 | "-v", "--verbose", dest="verbose", action="count", help="Verbose operation" |
| 114 | ) |
| 115 | parser.add_argument( |
| 116 | "-j", "--jobs", dest="jobs", type=int, default=1, help="Number of parallel jobs" |
| 117 | ) |
| 118 | parser.add_argument( |
| 119 | "--sut-module", |
| 120 | "-s", |
| 121 | dest="sut_module", |
| 122 | type=str, |
| 123 | nargs="+", |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 124 | default=[ttp.TOSA_REFMODEL_RUNNER], |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 125 | help="System under test module to load (derives from TosaTestRunner). May be repeated", |
| 126 | ) |
| 127 | parser.add_argument( |
| 128 | "--sut-module-args", |
| 129 | dest="sut_module_args", |
| 130 | type=str, |
| 131 | nargs="+", |
| 132 | default=[], |
| 133 | help="System under test module arguments. Use sutmodulename:argvalue to pass an argument. May be repeated.", |
| 134 | ) |
| 135 | parser.add_argument( |
| 136 | "--xunit-file", |
| 137 | dest="xunit_file", |
| 138 | type=str, |
| 139 | default="result.xml", |
| 140 | help="XUnit output file", |
| 141 | ) |
| 142 | parser.add_argument( |
| 143 | "--test-type", |
| 144 | dest="test_type", |
| 145 | type=str, |
| 146 | default="both", |
| 147 | choices=["positive", "negative", "both"], |
Jeremy Johnson | 8858862 | 2022-07-12 16:42:29 +0100 | [diff] [blame] | 148 | help="Filter tests based on expected failure status", |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 149 | ) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 150 | parser.add_argument( |
| 151 | "--no-color", |
| 152 | "--no-colour", |
| 153 | dest="no_color", |
| 154 | action="store_true", |
| 155 | help="Disable color output", |
| 156 | ) |
Jeremy Johnson | 8858862 | 2022-07-12 16:42:29 +0100 | [diff] [blame] | 157 | parser.add_argument( |
| 158 | "--profile", |
| 159 | dest="profile", |
| 160 | type=str, |
| 161 | choices=["tosa-bi", "tosa-mi"], |
| 162 | help="Filter tests based on profile", |
| 163 | ) |
Jerry Ge | a793f46 | 2023-04-11 00:05:02 +0000 | [diff] [blame] | 164 | parser.add_argument( |
| 165 | "--tosa_level", |
| 166 | dest="tosa_level", |
| 167 | default="EIGHTK", |
| 168 | type=str, |
| 169 | help="A TOSA level defines operator parameter ranges that an implementation shall support." |
| 170 | "Config tosa_level for running the reference model only. Default is EIGHTK", |
| 171 | ) |
Tai Ly | a4d748b | 2023-03-28 22:06:56 +0000 | [diff] [blame] | 172 | parser.add_argument( |
| 173 | "-p", |
| 174 | "--precise-mode", |
| 175 | dest="precise_mode", |
| 176 | action="store_true", |
| 177 | help="Run the reference model in precise mode (FP64)", |
| 178 | ) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 179 | |
| 180 | args = parser.parse_args(argv) |
| 181 | |
| 182 | # Autodetect CPU count |
| 183 | if args.jobs <= 0: |
| 184 | args.jobs = os.cpu_count() |
| 185 | |
| 186 | return args |
| 187 | |
| 188 | |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 189 | def workerThread(task_queue, runnerList, complianceRunner, args, result_queue): |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 190 | """Worker thread that runs the next test from the queue.""" |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 191 | complianceRunnerList = runnerList.copy() |
| 192 | complianceRunnerList.insert(0, (complianceRunner, [])) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 193 | while True: |
| 194 | try: |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 195 | test_path = task_queue.get(block=False) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 196 | except queue.Empty: |
| 197 | break |
| 198 | |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 199 | if test_path is None: |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 200 | break |
| 201 | |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 202 | try: |
| 203 | # Check for compliance test |
| 204 | desc = test_path / "desc.json" |
| 205 | with desc.open("r") as fd: |
| 206 | j = json.load(fd) |
| 207 | compliance = "compliance" in j["meta"] |
| 208 | except Exception: |
| 209 | compliance = False |
| 210 | |
| 211 | if compliance: |
| 212 | # Run compliance first to create output files! |
| 213 | currentRunners = complianceRunnerList |
| 214 | else: |
| 215 | currentRunners = runnerList |
| 216 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 217 | msg = "" |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 218 | for runnerModule, runnerArgs in currentRunners: |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 219 | try: |
| 220 | start_time = datetime.now() |
| 221 | # Set up system under test runner |
| 222 | runnerName = runnerModule.__name__ |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 223 | runner = runnerModule.TosaSUTRunner(args, runnerArgs, test_path) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 224 | |
Jeremy Johnson | 8858862 | 2022-07-12 16:42:29 +0100 | [diff] [blame] | 225 | skip, reason = runner.skipTest() |
| 226 | if skip: |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 227 | msg = f"Skipping {reason} test" |
| 228 | print(f"{msg} {test_path}") |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 229 | rc = TosaTestRunner.Result.SKIPPED |
| 230 | else: |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 231 | if args.verbose: |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 232 | print(f"Running runner {runnerName} with test {test_path}") |
| 233 | try: |
| 234 | # Convert or generate the required data files |
| 235 | runner.readyDataFiles() |
| 236 | except Exception as e: |
| 237 | msg = f"Failed to ready test files error: {e}" |
| 238 | raise e |
| 239 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 240 | try: |
| 241 | grc, gmsg = runner.runTestGraph() |
| 242 | rc, msg = runner.testResult(grc, gmsg) |
| 243 | except Exception as e: |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 244 | msg = f"System Under Test error: {e}" |
| 245 | raise e |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 246 | except Exception as e: |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 247 | if not msg: |
| 248 | msg = f"Internal error: {e}" |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 249 | print(msg) |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 250 | if not isinstance(e, (TosaTestInvalid, GenerateError)): |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 251 | # Show stack trace on unexpected exceptions |
| 252 | print( |
| 253 | "".join( |
| 254 | traceback.format_exception( |
| 255 | etype=type(e), value=e, tb=e.__traceback__ |
| 256 | ) |
| 257 | ) |
| 258 | ) |
| 259 | rc = TosaTestRunner.Result.INTERNAL_ERROR |
| 260 | finally: |
| 261 | end_time = datetime.now() |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 262 | result_queue.put( |
| 263 | (runnerName, test_path, rc, msg, end_time - start_time) |
| 264 | ) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 265 | |
| 266 | task_queue.task_done() |
| 267 | |
| 268 | return True |
| 269 | |
| 270 | |
| 271 | def loadSUTRunnerModules(args): |
| 272 | """Load in the system under test modules. |
| 273 | |
| 274 | Returns a list of tuples of (runner_module, [argument list]) |
| 275 | """ |
| 276 | runnerList = [] |
| 277 | # Remove any duplicates from the list |
| 278 | sut_module_list = list(set(args.sut_module)) |
| 279 | for r in sut_module_list: |
| 280 | if args.verbose: |
| 281 | print("Loading module {}".format(r)) |
| 282 | |
| 283 | runner = importlib.import_module(r) |
| 284 | |
| 285 | # Look for arguments associated with this runner |
| 286 | runnerArgPrefix = "{}:".format(r) |
| 287 | runnerArgList = [] |
| 288 | for a in args.sut_module_args: |
| 289 | if a.startswith(runnerArgPrefix): |
| 290 | runnerArgList.append(a[len(runnerArgPrefix) :]) |
| 291 | runnerList.append((runner, runnerArgList)) |
| 292 | |
| 293 | return runnerList |
| 294 | |
| 295 | |
| 296 | def createXUnitResults(xunitFile, runnerList, resultLists, verbose): |
| 297 | """Create the xunit results file.""" |
| 298 | xunit_result = xunit.xunit_results() |
| 299 | |
| 300 | for runnerModule, _ in runnerList: |
| 301 | # Create test suite per system under test (runner) |
| 302 | runner = runnerModule.__name__ |
| 303 | xunit_suite = xunit_result.create_suite(runner) |
| 304 | |
| 305 | # Sort by test name |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 306 | for test_path, rc, msg, time_delta in sorted( |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 307 | resultLists[runner], key=lambda tup: tup[0] |
| 308 | ): |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 309 | test_name = str(test_path) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 310 | xt = xunit.xunit_test(test_name, runner) |
| 311 | |
| 312 | xt.time = str( |
| 313 | float(time_delta.seconds) + (float(time_delta.microseconds) * 1e-6) |
| 314 | ) |
| 315 | |
| 316 | testMsg = rc.name if not msg else "{}: {}".format(rc.name, msg) |
| 317 | |
| 318 | if ( |
| 319 | rc == TosaTestRunner.Result.EXPECTED_PASS |
| 320 | or rc == TosaTestRunner.Result.EXPECTED_FAILURE |
| 321 | ): |
| 322 | if verbose: |
| 323 | print("{} {} ({})".format(rc.name, test_name, runner)) |
| 324 | elif rc == TosaTestRunner.Result.SKIPPED: |
| 325 | xt.skipped() |
| 326 | if verbose: |
| 327 | print("{} {} ({})".format(rc.name, test_name, runner)) |
| 328 | else: |
| 329 | xt.failed(testMsg) |
| 330 | print("{} {} ({})".format(rc.name, test_name, runner)) |
| 331 | |
| 332 | xunit_suite.tests.append(xt) |
| 333 | |
| 334 | xunit_result.write_results(xunitFile) |
| 335 | |
| 336 | |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 337 | def getTestsInPath(path): |
| 338 | # Recursively find any tests in this directory |
| 339 | desc_path = path / "desc.json" |
| 340 | if desc_path.is_file(): |
| 341 | return [path] |
| 342 | elif path.is_dir(): |
| 343 | path_list = [] |
| 344 | for p in path.glob("*"): |
| 345 | path_list.extend(getTestsInPath(p)) |
| 346 | return path_list |
| 347 | else: |
| 348 | return [] |
| 349 | |
| 350 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 351 | def main(argv=None): |
| 352 | """Start worker threads to do the testing and outputs the results.""" |
| 353 | args = parseArgs(argv) |
| 354 | |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 355 | # Set up some defaults |
| 356 | if args.ref_model_path is None: |
| 357 | args.ref_model_path = cmf.find_tosa_file( |
| 358 | cmf.TosaFileType.REF_MODEL, Path("reference_model"), False |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 359 | ) |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 360 | if args.generate_lib_path is None: |
| 361 | args.generate_lib_path = cmf.find_tosa_file( |
| 362 | cmf.TosaFileType.GENERATE_LIBRARY, args.ref_model_path |
| 363 | ) |
Jeremy Johnson | 9c2fe6e | 2023-10-04 16:55:04 +0100 | [diff] [blame] | 364 | if args.verify_lib_path is None: |
| 365 | args.verify_lib_path = cmf.find_tosa_file( |
| 366 | cmf.TosaFileType.VERIFY_LIBRARY, args.ref_model_path |
| 367 | ) |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 368 | if args.flatc_path is None: |
| 369 | args.flatc_path = cmf.find_tosa_file( |
| 370 | cmf.TosaFileType.FLATC, args.ref_model_path |
| 371 | ) |
| 372 | if args.schema_path is None: |
| 373 | args.schema_path = cmf.find_tosa_file( |
| 374 | cmf.TosaFileType.SCHEMA, args.ref_model_path |
| 375 | ) |
| 376 | |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 377 | # Always check as it will be needed for compliance |
| 378 | if not args.ref_model_path.is_file(): |
| 379 | print( |
| 380 | f"Argument error: Reference Model not found - ({str(args.ref_model_path)})" |
| 381 | ) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 382 | exit(2) |
| 383 | |
| 384 | if args.test_list_file: |
| 385 | try: |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 386 | with args.test_list_file.open("r") as f: |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 387 | args.test = f.read().splitlines() |
| 388 | except Exception as e: |
| 389 | print( |
| 390 | "Argument error: Cannot read list of tests in {}\n{}".format( |
| 391 | args.test_list_file, e |
| 392 | ) |
| 393 | ) |
| 394 | exit(2) |
| 395 | |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 396 | # Load in the runner modules and the ref model compliance module |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 397 | runnerList = loadSUTRunnerModules(args) |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 398 | complianceRunner = importlib.import_module(ttp.TOSA_REFCOMPLIANCE_RUNNER) |
| 399 | # Create a separate reporting runner list as the compliance runner may not |
| 400 | # be always run - depends on compliance testing |
| 401 | fullRunnerList = runnerList + [(complianceRunner, [])] |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 402 | |
| 403 | threads = [] |
| 404 | taskQueue = queue.Queue() |
| 405 | resultQueue = queue.Queue() |
| 406 | |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 407 | for tdir in args.test: |
| 408 | tpath = Path(tdir) |
| 409 | if tpath.is_file(): |
| 410 | if tpath.name != "README": |
| 411 | print( |
| 412 | "Warning: Skipping test {} as not a valid directory".format(tpath) |
| 413 | ) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 414 | else: |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 415 | if args.recursive_tests: |
| 416 | tpath_list = getTestsInPath(tpath) |
| 417 | else: |
| 418 | tpath_list = [tpath] |
| 419 | |
| 420 | for t in tpath_list: |
| 421 | taskQueue.put((t)) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 422 | |
| 423 | print( |
| 424 | "Running {} tests on {} system{} under test".format( |
| 425 | taskQueue.qsize(), len(runnerList), "s" if len(runnerList) > 1 else "" |
| 426 | ) |
| 427 | ) |
| 428 | |
| 429 | for i in range(args.jobs): |
| 430 | t = threading.Thread( |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 431 | target=workerThread, |
| 432 | args=(taskQueue, runnerList, complianceRunner, args, resultQueue), |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 433 | ) |
| 434 | t.setDaemon(True) |
| 435 | t.start() |
| 436 | threads.append(t) |
| 437 | |
| 438 | taskQueue.join() |
| 439 | |
| 440 | # Set up results lists for each system under test |
| 441 | resultLists = {} |
| 442 | results = {} |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 443 | for runnerModule, _ in fullRunnerList: |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 444 | runner = runnerModule.__name__ |
| 445 | resultLists[runner] = [] |
| 446 | results[runner] = [0] * len(TosaTestRunner.Result) |
| 447 | |
| 448 | while True: |
| 449 | try: |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 450 | runner, test_path, rc, msg, time_delta = resultQueue.get(block=False) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 451 | resultQueue.task_done() |
| 452 | except queue.Empty: |
| 453 | break |
| 454 | |
| 455 | # Limit error messages to make results easier to digest |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 456 | if msg and len(msg) > ttp.MAX_XUNIT_TEST_MESSAGE: |
| 457 | half = int(ttp.MAX_XUNIT_TEST_MESSAGE / 2) |
| 458 | trimmed = len(msg) - ttp.MAX_XUNIT_TEST_MESSAGE |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 459 | msg = "{} ...\nskipped {} bytes\n... {}".format( |
| 460 | msg[:half], trimmed, msg[-half:] |
| 461 | ) |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 462 | resultLists[runner].append((test_path, rc, msg, time_delta)) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 463 | results[runner][rc] += 1 |
| 464 | |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 465 | createXUnitResults(args.xunit_file, fullRunnerList, resultLists, args.verbose) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 466 | |
| 467 | # Print out results for each system under test |
Jeremy Johnson | e2b5e87 | 2023-09-14 17:02:09 +0100 | [diff] [blame] | 468 | for runnerModule, _ in fullRunnerList: |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 469 | runner = runnerModule.__name__ |
| 470 | resultSummary = [] |
| 471 | for result in TosaTestRunner.Result: |
| 472 | resultSummary.append( |
| 473 | "{} {}".format(results[runner][result], result.name.lower()) |
| 474 | ) |
| 475 | print("Totals ({}): {}".format(runner, ", ".join(resultSummary))) |
| 476 | |
| 477 | return 0 |
| 478 | |
| 479 | |
| 480 | if __name__ == "__main__": |
| 481 | exit(main()) |