Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 1 | """TOSA verification runner script.""" |
| 2 | # Copyright (c) 2020-2022, ARM Limited. |
| 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | import argparse |
| 5 | import glob |
| 6 | import importlib |
| 7 | import os |
| 8 | import queue |
| 9 | import threading |
| 10 | import traceback |
| 11 | from datetime import datetime |
| 12 | from pathlib import Path |
| 13 | |
| 14 | from json2numpy import json2numpy |
| 15 | from runner.tosa_test_runner import TosaTestInvalid |
| 16 | from runner.tosa_test_runner import TosaTestRunner |
| 17 | from xunit import xunit |
| 18 | |
| 19 | TOSA_REFMODEL_RUNNER = "runner.tosa_refmodel_sut_run" |
| 20 | MAX_XUNIT_TEST_MESSAGE = 1000 |
| 21 | |
| 22 | |
| 23 | def parseArgs(argv): |
| 24 | """Parse the arguments and return the settings.""" |
| 25 | parser = argparse.ArgumentParser() |
| 26 | group = parser.add_mutually_exclusive_group(required=True) |
| 27 | group.add_argument( |
| 28 | "-t", |
| 29 | "--test", |
| 30 | dest="test", |
| 31 | type=str, |
| 32 | nargs="+", |
| 33 | help="Test(s) to run", |
| 34 | ) |
| 35 | group.add_argument( |
| 36 | "-T", |
| 37 | "--test-list", |
| 38 | dest="test_list_file", |
| 39 | type=Path, |
| 40 | help="File containing list of tests to run (one per line)", |
| 41 | ) |
| 42 | parser.add_argument( |
| 43 | "--operator-fbs", |
| 44 | dest="operator_fbs", |
| 45 | default="conformance_tests/third_party/serialization_lib/schema/tosa.fbs", |
| 46 | type=str, |
| 47 | help="flat buffer syntax file", |
| 48 | ) |
| 49 | parser.add_argument( |
| 50 | "--ref-model-path", |
| 51 | dest="ref_model_path", |
| 52 | default="reference_model/build/reference_model/tosa_reference_model", |
| 53 | type=str, |
| 54 | help="Path to reference model executable", |
| 55 | ) |
| 56 | parser.add_argument( |
| 57 | "--flatc-path", |
| 58 | dest="flatc_path", |
| 59 | default="reference_model/build/thirdparty/serialization_lib/third_party/flatbuffers/flatc", |
| 60 | type=str, |
| 61 | help="Path to flatc compiler executable", |
| 62 | ) |
| 63 | parser.add_argument( |
| 64 | "--ref-debug", |
| 65 | dest="ref_debug", |
| 66 | default="", |
| 67 | type=str, |
| 68 | help="Reference debug flag (low, med, high)", |
| 69 | ) |
| 70 | parser.add_argument( |
| 71 | "--ref-intermediates", |
| 72 | dest="ref_intermediates", |
| 73 | default=0, |
| 74 | type=int, |
| 75 | help="Reference model dumps intermediate tensors", |
| 76 | ) |
| 77 | parser.add_argument( |
| 78 | "-b", |
| 79 | "--binary", |
| 80 | dest="binary", |
| 81 | action="store_true", |
| 82 | help="Convert to using binary flatbuffers instead of JSON", |
| 83 | ) |
| 84 | parser.add_argument( |
| 85 | "-v", "--verbose", dest="verbose", action="count", help="Verbose operation" |
| 86 | ) |
| 87 | parser.add_argument( |
| 88 | "-j", "--jobs", dest="jobs", type=int, default=1, help="Number of parallel jobs" |
| 89 | ) |
| 90 | parser.add_argument( |
| 91 | "--sut-module", |
| 92 | "-s", |
| 93 | dest="sut_module", |
| 94 | type=str, |
| 95 | nargs="+", |
| 96 | default=[TOSA_REFMODEL_RUNNER], |
| 97 | help="System under test module to load (derives from TosaTestRunner). May be repeated", |
| 98 | ) |
| 99 | parser.add_argument( |
| 100 | "--sut-module-args", |
| 101 | dest="sut_module_args", |
| 102 | type=str, |
| 103 | nargs="+", |
| 104 | default=[], |
| 105 | help="System under test module arguments. Use sutmodulename:argvalue to pass an argument. May be repeated.", |
| 106 | ) |
| 107 | parser.add_argument( |
| 108 | "--xunit-file", |
| 109 | dest="xunit_file", |
| 110 | type=str, |
| 111 | default="result.xml", |
| 112 | help="XUnit output file", |
| 113 | ) |
| 114 | parser.add_argument( |
| 115 | "--test-type", |
| 116 | dest="test_type", |
| 117 | type=str, |
| 118 | default="both", |
| 119 | choices=["positive", "negative", "both"], |
| 120 | help="Filter tests based on expected failure status (positive, negative or both)", |
| 121 | ) |
| 122 | |
| 123 | args = parser.parse_args(argv) |
| 124 | |
| 125 | # Autodetect CPU count |
| 126 | if args.jobs <= 0: |
| 127 | args.jobs = os.cpu_count() |
| 128 | |
| 129 | return args |
| 130 | |
| 131 | |
| 132 | EXCLUSION_PREFIX = ["test", "model", "desc"] |
| 133 | |
| 134 | |
| 135 | def convert2Numpy(testDir): |
| 136 | """Convert all the JSON numpy files back into binary numpy.""" |
| 137 | jsons = glob.glob(os.path.join(testDir, "*.json")) |
| 138 | for json in jsons: |
| 139 | for exclude in EXCLUSION_PREFIX: |
| 140 | if os.path.basename(json).startswith(exclude): |
| 141 | json = "" |
| 142 | if json: |
| 143 | # debug print("Converting " + json) |
| 144 | json2numpy.json_to_npy(Path(json)) |
| 145 | |
| 146 | |
| 147 | def workerThread(task_queue, runnerList, args, result_queue): |
| 148 | """Worker thread that runs the next test from the queue.""" |
| 149 | while True: |
| 150 | try: |
| 151 | test = task_queue.get(block=False) |
| 152 | except queue.Empty: |
| 153 | break |
| 154 | |
| 155 | if test is None: |
| 156 | break |
| 157 | |
| 158 | msg = "" |
| 159 | converted = False |
| 160 | for runnerModule, runnerArgs in runnerList: |
| 161 | try: |
| 162 | start_time = datetime.now() |
| 163 | # Set up system under test runner |
| 164 | runnerName = runnerModule.__name__ |
| 165 | runner = runnerModule.TosaSUTRunner(args, runnerArgs, test) |
| 166 | |
| 167 | if runner.skipTest(): |
| 168 | msg = "Skipping non-{} test".format(args.test_type) |
| 169 | print("{} {}".format(msg, test)) |
| 170 | rc = TosaTestRunner.Result.SKIPPED |
| 171 | else: |
| 172 | # Convert JSON data files into numpy format on first pass |
| 173 | if not converted: |
| 174 | convert2Numpy(test) |
| 175 | converted = True |
| 176 | |
| 177 | if args.verbose: |
| 178 | print("Running runner {} with test {}".format(runnerName, test)) |
| 179 | try: |
| 180 | grc, gmsg = runner.runTestGraph() |
| 181 | rc, msg = runner.testResult(grc, gmsg) |
| 182 | except Exception as e: |
| 183 | msg = "System Under Test error: {}".format(e) |
| 184 | print(msg) |
| 185 | print( |
| 186 | "".join( |
| 187 | traceback.format_exception( |
| 188 | etype=type(e), value=e, tb=e.__traceback__ |
| 189 | ) |
| 190 | ) |
| 191 | ) |
| 192 | rc = TosaTestRunner.Result.INTERNAL_ERROR |
| 193 | except Exception as e: |
| 194 | msg = "Internal error: {}".format(e) |
| 195 | print(msg) |
| 196 | if not isinstance(e, TosaTestInvalid): |
| 197 | # Show stack trace on unexpected exceptions |
| 198 | print( |
| 199 | "".join( |
| 200 | traceback.format_exception( |
| 201 | etype=type(e), value=e, tb=e.__traceback__ |
| 202 | ) |
| 203 | ) |
| 204 | ) |
| 205 | rc = TosaTestRunner.Result.INTERNAL_ERROR |
| 206 | finally: |
| 207 | end_time = datetime.now() |
| 208 | result_queue.put((runnerName, test, rc, msg, end_time - start_time)) |
| 209 | |
| 210 | task_queue.task_done() |
| 211 | |
| 212 | return True |
| 213 | |
| 214 | |
| 215 | def loadSUTRunnerModules(args): |
| 216 | """Load in the system under test modules. |
| 217 | |
| 218 | Returns a list of tuples of (runner_module, [argument list]) |
| 219 | """ |
| 220 | runnerList = [] |
| 221 | # Remove any duplicates from the list |
| 222 | sut_module_list = list(set(args.sut_module)) |
| 223 | for r in sut_module_list: |
| 224 | if args.verbose: |
| 225 | print("Loading module {}".format(r)) |
| 226 | |
| 227 | runner = importlib.import_module(r) |
| 228 | |
| 229 | # Look for arguments associated with this runner |
| 230 | runnerArgPrefix = "{}:".format(r) |
| 231 | runnerArgList = [] |
| 232 | for a in args.sut_module_args: |
| 233 | if a.startswith(runnerArgPrefix): |
| 234 | runnerArgList.append(a[len(runnerArgPrefix) :]) |
| 235 | runnerList.append((runner, runnerArgList)) |
| 236 | |
| 237 | return runnerList |
| 238 | |
| 239 | |
| 240 | def createXUnitResults(xunitFile, runnerList, resultLists, verbose): |
| 241 | """Create the xunit results file.""" |
| 242 | xunit_result = xunit.xunit_results() |
| 243 | |
| 244 | for runnerModule, _ in runnerList: |
| 245 | # Create test suite per system under test (runner) |
| 246 | runner = runnerModule.__name__ |
| 247 | xunit_suite = xunit_result.create_suite(runner) |
| 248 | |
| 249 | # Sort by test name |
| 250 | for test, rc, msg, time_delta in sorted( |
| 251 | resultLists[runner], key=lambda tup: tup[0] |
| 252 | ): |
| 253 | test_name = test |
| 254 | xt = xunit.xunit_test(test_name, runner) |
| 255 | |
| 256 | xt.time = str( |
| 257 | float(time_delta.seconds) + (float(time_delta.microseconds) * 1e-6) |
| 258 | ) |
| 259 | |
| 260 | testMsg = rc.name if not msg else "{}: {}".format(rc.name, msg) |
| 261 | |
| 262 | if ( |
| 263 | rc == TosaTestRunner.Result.EXPECTED_PASS |
| 264 | or rc == TosaTestRunner.Result.EXPECTED_FAILURE |
| 265 | ): |
| 266 | if verbose: |
| 267 | print("{} {} ({})".format(rc.name, test_name, runner)) |
| 268 | elif rc == TosaTestRunner.Result.SKIPPED: |
| 269 | xt.skipped() |
| 270 | if verbose: |
| 271 | print("{} {} ({})".format(rc.name, test_name, runner)) |
| 272 | else: |
| 273 | xt.failed(testMsg) |
| 274 | print("{} {} ({})".format(rc.name, test_name, runner)) |
| 275 | |
| 276 | xunit_suite.tests.append(xt) |
| 277 | |
| 278 | xunit_result.write_results(xunitFile) |
| 279 | |
| 280 | |
| 281 | def main(argv=None): |
| 282 | """Start worker threads to do the testing and outputs the results.""" |
| 283 | args = parseArgs(argv) |
| 284 | |
| 285 | if TOSA_REFMODEL_RUNNER in args.sut_module and not os.path.isfile( |
| 286 | args.ref_model_path |
| 287 | ): |
| 288 | print( |
| 289 | "Argument error: Reference Model not found ({})".format(args.ref_model_path) |
| 290 | ) |
| 291 | exit(2) |
| 292 | |
| 293 | if args.test_list_file: |
| 294 | try: |
| 295 | with open(args.test_list_file) as f: |
| 296 | args.test = f.read().splitlines() |
| 297 | except Exception as e: |
| 298 | print( |
| 299 | "Argument error: Cannot read list of tests in {}\n{}".format( |
| 300 | args.test_list_file, e |
| 301 | ) |
| 302 | ) |
| 303 | exit(2) |
| 304 | |
| 305 | runnerList = loadSUTRunnerModules(args) |
| 306 | |
| 307 | threads = [] |
| 308 | taskQueue = queue.Queue() |
| 309 | resultQueue = queue.Queue() |
| 310 | |
| 311 | for t in args.test: |
| 312 | if os.path.isfile(t): |
| 313 | if not os.path.basename(t) == "README": |
| 314 | print("Warning: Skipping test {} as not a valid directory".format(t)) |
| 315 | else: |
| 316 | taskQueue.put((t)) |
| 317 | |
| 318 | print( |
| 319 | "Running {} tests on {} system{} under test".format( |
| 320 | taskQueue.qsize(), len(runnerList), "s" if len(runnerList) > 1 else "" |
| 321 | ) |
| 322 | ) |
| 323 | |
| 324 | for i in range(args.jobs): |
| 325 | t = threading.Thread( |
| 326 | target=workerThread, args=(taskQueue, runnerList, args, resultQueue) |
| 327 | ) |
| 328 | t.setDaemon(True) |
| 329 | t.start() |
| 330 | threads.append(t) |
| 331 | |
| 332 | taskQueue.join() |
| 333 | |
| 334 | # Set up results lists for each system under test |
| 335 | resultLists = {} |
| 336 | results = {} |
| 337 | for runnerModule, _ in runnerList: |
| 338 | runner = runnerModule.__name__ |
| 339 | resultLists[runner] = [] |
| 340 | results[runner] = [0] * len(TosaTestRunner.Result) |
| 341 | |
| 342 | while True: |
| 343 | try: |
| 344 | runner, test, rc, msg, time_delta = resultQueue.get(block=False) |
| 345 | resultQueue.task_done() |
| 346 | except queue.Empty: |
| 347 | break |
| 348 | |
| 349 | # Limit error messages to make results easier to digest |
| 350 | if msg and len(msg) > MAX_XUNIT_TEST_MESSAGE: |
| 351 | half = int(MAX_XUNIT_TEST_MESSAGE / 2) |
| 352 | trimmed = len(msg) - MAX_XUNIT_TEST_MESSAGE |
| 353 | msg = "{} ...\nskipped {} bytes\n... {}".format( |
| 354 | msg[:half], trimmed, msg[-half:] |
| 355 | ) |
| 356 | resultLists[runner].append((test, rc, msg, time_delta)) |
| 357 | results[runner][rc] += 1 |
| 358 | |
| 359 | createXUnitResults(args.xunit_file, runnerList, resultLists, args.verbose) |
| 360 | |
| 361 | # Print out results for each system under test |
| 362 | for runnerModule, _ in runnerList: |
| 363 | runner = runnerModule.__name__ |
| 364 | resultSummary = [] |
| 365 | for result in TosaTestRunner.Result: |
| 366 | resultSummary.append( |
| 367 | "{} {}".format(results[runner][result], result.name.lower()) |
| 368 | ) |
| 369 | print("Totals ({}): {}".format(runner, ", ".join(resultSummary))) |
| 370 | |
| 371 | return 0 |
| 372 | |
| 373 | |
| 374 | if __name__ == "__main__": |
| 375 | exit(main()) |