Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Luke Hutton | 261b7b6 | 2023-01-10 14:50:31 +0000 | [diff] [blame] | 2 | # Copyright (c) 2020-2023, ARM Limited. |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | import argparse |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 5 | import json |
| 6 | import math |
| 7 | import os |
| 8 | import queue |
| 9 | import re |
| 10 | import sys |
| 11 | import threading |
| 12 | import traceback |
| 13 | from datetime import datetime |
| 14 | from enum import IntEnum |
| 15 | from enum import unique |
Eric Kunze | 97b0027 | 2023-07-20 10:52:56 -0700 | [diff] [blame] | 16 | from pathlib import Path |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 17 | |
| 18 | import numpy as np |
| 19 | from checker.tosa_result_checker import LogColors |
| 20 | from checker.tosa_result_checker import print_color |
| 21 | from checker.tosa_result_checker import set_print_in_color |
| 22 | from runner.run_command import run_sh_command |
| 23 | from xunit.xunit import xunit_results |
| 24 | from xunit.xunit import xunit_test |
| 25 | |
| 26 | |
| 27 | def parse_args(): |
| 28 | parser = argparse.ArgumentParser() |
| 29 | parser.add_argument( |
Jared Smolens | b7af461 | 2022-03-21 19:41:52 -0700 | [diff] [blame] | 30 | "-t", |
| 31 | "--test", |
| 32 | dest="test", |
| 33 | default=[], |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 34 | type=Path, |
Jared Smolens | b7af461 | 2022-03-21 19:41:52 -0700 | [diff] [blame] | 35 | nargs="+", |
| 36 | help="Test(s) to run", |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 37 | ) |
| 38 | parser.add_argument( |
| 39 | "-r", |
| 40 | "--recursive", |
| 41 | dest="recursive_tests", |
| 42 | action="store_true", |
| 43 | help="Recursively search for tests", |
| 44 | ) |
| 45 | parser.add_argument( |
| 46 | "--tf-base-dir", |
| 47 | dest="tf_base_dir", |
| 48 | type=str, |
| 49 | required=True, |
| 50 | help="Tensorflow/MLIR base directory", |
| 51 | ) |
| 52 | parser.add_argument( |
| 53 | "--tools-base-dir", |
| 54 | dest="tools_base_dir", |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 55 | type=Path, |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 56 | required=True, |
| 57 | help="Reference model base directory", |
| 58 | ) |
| 59 | parser.add_argument( |
Tai Ly | a4d748b | 2023-03-28 22:06:56 +0000 | [diff] [blame] | 60 | "-p", |
| 61 | "--precise-mode", |
| 62 | dest="precise_mode", |
| 63 | action="store_true", |
| 64 | help="run in precise mode (FP64)", |
| 65 | ) |
| 66 | parser.add_argument( |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 67 | "-v", "--verbose", dest="verbose", action="count", help="Verbose run" |
| 68 | ) |
| 69 | parser.add_argument( |
| 70 | "-dref", |
| 71 | "--debug-ref-model", |
| 72 | dest="debug_ref_model", |
| 73 | action="store_true", |
| 74 | help="Enable TOSA Reference model debugging", |
| 75 | ) |
| 76 | parser.add_argument( |
| 77 | "--tolerance", |
| 78 | dest="tolerance", |
| 79 | default=1e-3, |
| 80 | type=float, |
| 81 | help="Comparison tolerance b value", |
| 82 | ) |
| 83 | parser.add_argument( |
Jerry Ge | a793f46 | 2023-04-11 00:05:02 +0000 | [diff] [blame] | 84 | "--tosa_level", |
| 85 | dest="tosa_level", |
| 86 | default="EIGHTK", |
| 87 | type=str, |
| 88 | help="A TOSA level defines operator parameter ranges that an implementation shall support." |
| 89 | "Config tosa_level for running the reference model only. Default is EIGHTK", |
| 90 | ) |
| 91 | parser.add_argument( |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 92 | "--no-compiler", |
| 93 | dest="no_compiler", |
| 94 | action="store_true", |
| 95 | help="Do not run TF MLIR/tfopt/TOSA compiler. Just run TOSA Reference model", |
| 96 | ) |
| 97 | parser.add_argument( |
| 98 | "--no-ref-model", |
| 99 | dest="no_ref", |
| 100 | action="store_true", |
| 101 | help="Do not run TOSA reference model, just run TF MLIR/tfopt/TOSA compiler.", |
| 102 | ) |
| 103 | parser.add_argument( |
| 104 | "--valgrind", |
| 105 | dest="valgrind", |
| 106 | action="store_true", |
| 107 | help="Enable valgrind on TOSA Reference Model", |
| 108 | ) |
| 109 | parser.add_argument( |
| 110 | "-j", "--jobs", dest="jobs", type=int, default=1, help="Number of parallel jobs" |
| 111 | ) |
| 112 | parser.add_argument( |
| 113 | "--no-color", |
| 114 | "--no-colour", |
| 115 | dest="no_color", |
| 116 | action="store_true", |
| 117 | help="Disable color output", |
| 118 | ) |
| 119 | parser.add_argument( |
| 120 | "-f", |
| 121 | "--framework", |
| 122 | dest="framework", |
| 123 | default=[], |
| 124 | action="append", |
| 125 | help="Frameworks to test (tf, tflite)", |
| 126 | ) |
| 127 | parser.add_argument( |
| 128 | "--override-exclusions", |
| 129 | dest="override_exclusions", |
| 130 | default=False, |
| 131 | action="store_true", |
| 132 | help="Ignore the framework exclusions listed in the test JSON", |
| 133 | ) |
| 134 | parser.add_argument( |
| 135 | "--xunit-file", |
| 136 | dest="xunit_file", |
| 137 | type=str, |
| 138 | default="result.xml", |
| 139 | help="XUnit result output file", |
| 140 | ) |
| 141 | parser.add_argument( |
| 142 | "--xunit-classname-prefix", |
| 143 | dest="xunit_classname_prefix", |
| 144 | default="TFUnitTests", |
| 145 | help="Prefix for xunit classname", |
| 146 | ) |
| 147 | parser.add_argument( |
| 148 | "--hex-bool-hack", |
| 149 | dest="hex_bool_hack", |
| 150 | default=1, |
| 151 | type=int, |
| 152 | help=( |
| 153 | "Hack around bug in MLIR hex parsing for boolean types" |
| 154 | " by disabling hex encoding" |
| 155 | ), |
| 156 | ) |
| 157 | parser.add_argument( |
| 158 | "--regression-mode", |
| 159 | dest="regression_mode", |
| 160 | default=False, |
| 161 | action="store_true", |
| 162 | help="Options to make the script more friendly for jenkins regressions", |
| 163 | ) |
| 164 | parser.add_argument( |
| 165 | "--quantize-tolerance", |
| 166 | dest="quantize_tolerance", |
| 167 | default=0, |
| 168 | type=int, |
| 169 | help=( |
| 170 | "Tolerance when comparing TOSA reference model result" |
| 171 | " to TensorFlow Lite reference" |
| 172 | ), |
| 173 | ) |
| 174 | parser.add_argument( |
| 175 | "--test-dir", |
| 176 | dest="test_dir", |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 177 | type=Path, |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 178 | help="Path to prepend to paths in test.json", |
| 179 | ) |
| 180 | |
| 181 | parser.add_argument( |
| 182 | "-o", "--output", dest="output_file", help="Redirect script output to a file" |
| 183 | ) |
| 184 | |
| 185 | args = parser.parse_args() |
| 186 | |
| 187 | # No easy way to both do array append and override a default value |
| 188 | if not args.framework: |
| 189 | args.framework = ["tf", "tflite"] |
| 190 | |
| 191 | # Autodetect CPU count |
| 192 | if args.jobs <= 0: |
| 193 | args.jobs = os.cpu_count() |
| 194 | |
| 195 | return args |
| 196 | |
| 197 | |
| 198 | @unique |
| 199 | class TestResult(IntEnum): |
| 200 | PASS = 0 |
| 201 | COMPILER_ERROR = 1 |
| 202 | REF_MODEL_ERROR = 2 |
| 203 | REF_MODEL_UNPREDICTABLE = 3 |
| 204 | REF_MODEL_RUNTIME_ERROR = 4 |
| 205 | MISMATCH = 5 |
| 206 | NOT_LOWERED = 6 |
| 207 | INVALID_MLIR = 7 |
| 208 | INTERNAL_ERROR = 8 |
| 209 | SKIPPED = 9 |
| 210 | |
| 211 | |
| 212 | TestResultErrorStr = [ |
| 213 | "", |
| 214 | "Compiler error", |
| 215 | "Reference model error", |
| 216 | "Reference model unpredictable", |
| 217 | "Reference model runtime error", |
| 218 | "Mismatch", |
| 219 | "Not lowered", |
| 220 | "Invalid MLIR", |
| 221 | "Internal error", |
| 222 | "", |
| 223 | ] |
| 224 | |
| 225 | |
| 226 | def parse_compiler_output(compiler_stdout, compiler_stderr): |
| 227 | # Look for "has not been lowered yet, skipped" strings in stdout |
| 228 | expr = re.compile(".* has not been lowered yet, skipped.*") |
| 229 | |
| 230 | for line in compiler_stdout.splitlines(): |
| 231 | if expr.match(line): |
| 232 | return TestResult.NOT_LOWERED |
| 233 | |
| 234 | return TestResult.PASS |
| 235 | |
| 236 | |
| 237 | def parse_reference_model_output(ref_model_stdout, ref_model_stderr): |
| 238 | # Look for "has not been lowered yet, skipped" strings in stdout |
| 239 | unpredictable_expr = re.compile(r".*UNPREDICTABLE.*") |
| 240 | error_expr = re.compile(".* Graph result: ERROR.*") |
| 241 | unknown_expr = re.compile(".* Unknown graph status code.*") |
| 242 | |
| 243 | for line in ref_model_stderr.splitlines(): |
| 244 | if unpredictable_expr.match(line): |
| 245 | return TestResult.REF_MODEL_UNPREDICTABLE |
| 246 | elif error_expr.match(line): |
| 247 | return TestResult.REF_MODEL_ERROR |
| 248 | elif unknown_expr.match(line): |
| 249 | return TestResult.REF_MODEL_RUNTIME_ERROR |
| 250 | |
| 251 | return TestResult.PASS |
| 252 | |
| 253 | |
| 254 | # write a self-contained test descriptor in json format |
| 255 | def write_reference_runner_json( |
| 256 | filename, |
| 257 | tosa_filename, |
| 258 | ifm_name, |
| 259 | ifm_file, |
| 260 | ofm_name, |
| 261 | ofm_file, |
| 262 | expected_failure=False, |
| 263 | ): |
| 264 | """Write a json test file so that it is fairly easy to pick up the test |
| 265 | and generate commands for third party tool""" |
| 266 | test_desc = dict() |
| 267 | |
| 268 | test_desc["tosa_file"] = tosa_filename |
| 269 | test_desc["ifm_name"] = ifm_name |
| 270 | test_desc["ifm_file"] = ifm_file |
| 271 | test_desc["ofm_name"] = ofm_name |
| 272 | test_desc["ofm_file"] = ofm_file |
| 273 | test_desc["expected_failure"] = expected_failure |
| 274 | |
| 275 | with open(filename, "w") as f: |
| 276 | json.dump(test_desc, f, indent=" ") |
| 277 | |
| 278 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 279 | def run_test(args, test_path, framework): |
Eric Kunze | 97b0027 | 2023-07-20 10:52:56 -0700 | [diff] [blame] | 280 | msg = "" |
| 281 | |
| 282 | try: |
| 283 | with open(test_path / "test.json", "r") as f: |
| 284 | test_desc = json.load(f) |
| 285 | except Exception: |
| 286 | raise Exception(f"Could not load or parse test from {test_path / 'test.json'}") |
| 287 | |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 288 | test_name = None |
Eric Kunze | 97b0027 | 2023-07-20 10:52:56 -0700 | [diff] [blame] | 289 | if "name" in test_desc: |
| 290 | test_name = test_desc["name"] |
| 291 | else: |
| 292 | test_name = test_path.name |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 293 | if not test_name: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 294 | raise Exception(f"Could not parse test_name from {test_path}") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 295 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 296 | print_color(LogColors.GREEN, f"## Running {framework} test {test_name}") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 297 | |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 298 | try: |
| 299 | if not args.override_exclusions: |
| 300 | for excl in test_desc["framework_exclusions"]: |
| 301 | if excl == framework: |
| 302 | print_color(LogColors.GREEN, "Results SKIPPED") |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 303 | return (TestResult.SKIPPED, 0.0, "", test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 304 | except KeyError: |
| 305 | pass |
| 306 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 307 | tf_tools_dir = Path( |
| 308 | f"{args.tf_base_dir}/bazel-bin/tensorflow/compiler/mlir" |
| 309 | ).resolve() |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 310 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 311 | pre_opt_filename = str(test_path / f"test_{framework}.preopt.mlir") |
| 312 | post_opt_filename = str(test_path / f"test_{framework}.postopt.mlir") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 313 | if args.test_dir: |
| 314 | test_path_prepend = args.test_dir |
| 315 | else: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 316 | test_path_prepend = test_path |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 317 | |
| 318 | # 1. Framework to MLIR translator command |
| 319 | if framework == "tf": |
| 320 | if test_desc["tf_model_filename"].endswith(".mlir"): |
| 321 | pre_opt_filename = test_desc["tf_model_filename"] |
| 322 | translate_mlir_cmd = [] |
| 323 | else: |
| 324 | translate_mlir_cmd = [ |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 325 | str(tf_tools_dir / "tf-mlir-translate"), |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 326 | "--graphdef-to-mlir", |
| 327 | "--tf-enable-shape-inference-on-import", |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 328 | f"--tf-output-arrays={test_desc['tf_result_name']}", |
| 329 | str(test_path_prepend / test_desc["tf_model_filename"]), |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 330 | "-o", |
| 331 | pre_opt_filename, |
| 332 | ] |
| 333 | elif framework == "tflite": |
| 334 | if test_desc["tflite_model_filename"].endswith(".mlir"): |
| 335 | pre_opt_filename = test_desc["tflite_model_filename"] |
| 336 | translate_mlir_cmd = [] |
| 337 | else: |
| 338 | translate_mlir_cmd = [ |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 339 | str(tf_tools_dir / "lite" / "flatbuffer_translate"), |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 340 | "--tflite-flatbuffer-to-mlir", |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 341 | str(test_path_prepend / test_desc["tflite_model_filename"]), |
| 342 | f"--output-arrays={test_desc['tflite_result_name']}", |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 343 | "-o", |
| 344 | pre_opt_filename, |
| 345 | ] |
| 346 | else: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 347 | raise Exception(f"Unknown framwork: {framework}") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 348 | |
| 349 | # Any additional inputs to the translator? |
| 350 | input_tensor_prefix = "TosaInput_" |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 351 | flatbuffer_dir = f"flatbuffer-{framework}" |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 352 | mlir_opts = [] |
| 353 | |
| 354 | # Temporary hack: MLIR's new hex encoding of large tensors does not work for |
| 355 | # boolean types |
| 356 | # for TF hash 8e8041d594a888eb67eafa5cc62627d7e9ca8082 |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 357 | if str(test_path).endswith("_bool") and args.hex_bool_hack: |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 358 | mlir_opts.append("--mlir-print-elementsattrs-with-hex-if-larger=-1") |
| 359 | |
| 360 | try: |
| 361 | # specify input tensors if test is generated from .pb |
| 362 | if framework == "tf": |
| 363 | # Convert the shape to a mlir-friendly string |
| 364 | shapes = [] |
| 365 | for curr_shape in test_desc["ifm_shape"]: |
| 366 | shape_str = "" |
| 367 | for dim in curr_shape: |
| 368 | shape_str = shape_str + str(dim) + "," |
| 369 | shapes.append(shape_str) |
| 370 | |
| 371 | translate_mlir_cmd.extend( |
| 372 | ["--tf-input-arrays", ",".join(test_desc["ifm_name"])] |
| 373 | ) |
| 374 | translate_mlir_cmd.extend(["--tf-input-shapes", ":".join(shapes)]) |
| 375 | |
| 376 | # Write the hard-coded placeholder input (reshaped as necesary) to |
| 377 | # the file that compiler specified. |
| 378 | reference_runner_ifm_name = [] |
| 379 | for i in range(len(test_desc["ifm_file"])): |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 380 | ifm_tensor_name = f"{input_tensor_prefix}{i}" |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 381 | |
| 382 | assert test_desc["ifm_file"][i].endswith(".npy") |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 383 | ifm_np = np.load(test_path / test_desc["ifm_file"][i]) |
Jared Smolens | b7af461 | 2022-03-21 19:41:52 -0700 | [diff] [blame] | 384 | |
| 385 | # We sometimes encounter input shape/expected input shape mismatches |
| 386 | # due to a missing batch dimension on the input (e.g. a single 3D image). |
| 387 | # |
| 388 | # Make sure input numpy and input shape from descriptor match, |
| 389 | # expand_dims on the outer dimensions until the rank matches, |
| 390 | # then do the shape comparison. |
| 391 | while len(list(ifm_np.shape)) < len(test_desc["ifm_shape"][i]): |
| 392 | ifm_np = np.expand_dims(ifm_np, axis=0) |
| 393 | |
Luke Hutton | 714aa60 | 2023-02-08 19:45:26 +0000 | [diff] [blame] | 394 | # After legalization, complex tensors are expected to be represented |
| 395 | # as a single floating point tensor of shape [?, ..., ?, 2]. |
| 396 | expected_shape = test_desc["ifm_shape"][i] |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 397 | if str(test_path).endswith("c64"): |
Luke Hutton | 714aa60 | 2023-02-08 19:45:26 +0000 | [diff] [blame] | 398 | expected_shape.append(2) |
| 399 | |
| 400 | assert list(ifm_np.shape) == expected_shape |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 401 | |
| 402 | reference_runner_ifm_name.append(ifm_tensor_name) |
| 403 | |
| 404 | except KeyError: |
| 405 | # No additional inputs. Ignore. |
| 406 | pass |
| 407 | |
| 408 | tf_opt_cmd = [ |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 409 | str(tf_tools_dir / "tf-opt"), |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 410 | "--tf-executor-to-functional-conversion", |
| 411 | "--verify-each", |
| 412 | pre_opt_filename, |
| 413 | "-o", |
| 414 | post_opt_filename, |
| 415 | ] |
| 416 | |
| 417 | translate_mlir_cmd.extend(mlir_opts) |
| 418 | tf_opt_cmd.extend(mlir_opts) |
| 419 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 420 | compiler_cmd = [str(tf_tools_dir / "tf-opt")] |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 421 | |
| 422 | if framework == "tf": |
| 423 | compiler_cmd.append("--tf-to-tosa-pipeline") |
| 424 | elif framework == "tflite": |
| 425 | compiler_cmd.append("--tfl-to-tosa-pipeline") |
| 426 | compiler_cmd.append("--tosa-strip-quant-types") |
| 427 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 428 | tosa_mlir_filename = str(test_path / f"output_{framework}.tosa.mlir") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 429 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 430 | flatbuffer_dir_fullpath = test_path / flatbuffer_dir |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 431 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 432 | flatbuffer_dir_fullpath.mkdir(exist_ok=True) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 433 | |
| 434 | compiler_cmd.extend( |
| 435 | [ |
| 436 | "--verify-each", |
| 437 | post_opt_filename, |
| 438 | "-o", |
| 439 | tosa_mlir_filename, |
| 440 | "--tosa-serialize", |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 441 | f"--tosa-flatbuffer-filename={flatbuffer_dir_fullpath / f'{test_name}.tosa'}", |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 442 | ] |
| 443 | ) |
| 444 | |
| 445 | if not args.no_compiler: |
| 446 | try: |
| 447 | if translate_mlir_cmd: |
| 448 | run_sh_command(translate_mlir_cmd, args.verbose, True) |
| 449 | if tf_opt_cmd: |
| 450 | run_sh_command(tf_opt_cmd, args.verbose, True) |
| 451 | except Exception as e: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 452 | print_color(LogColors.RED, f"Results INVALID_MLIR {test_name}: {e}") |
| 453 | return (TestResult.INVALID_MLIR, 0.0, e, test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 454 | |
| 455 | try: |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 456 | compiler_stdout, compiler_stderr = run_sh_command( |
| 457 | compiler_cmd, args.verbose, True |
| 458 | ) |
| 459 | compiler_rc = parse_compiler_output(compiler_stdout, compiler_stderr) |
| 460 | if compiler_rc == TestResult.NOT_LOWERED: |
| 461 | print_color( |
| 462 | LogColors.RED, |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 463 | f"Results NOT_LOWERED {test_name}, framework {framework}", |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 464 | ) |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 465 | return (TestResult.NOT_LOWERED, 0.0, "", test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 466 | |
| 467 | pass |
| 468 | |
| 469 | except Exception as e: |
| 470 | if "same scale constraint" in str(e): |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 471 | print_color(LogColors.RED, f"Results INVALID_MLIR {test_name}: {e}") |
| 472 | return (TestResult.INVALID_MLIR, 0.0, e, test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 473 | else: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 474 | print_color(LogColors.RED, f"Results COMPILER_ERROR {test_name}: {e}") |
| 475 | return (TestResult.COMPILER_ERROR, 0.0, e, test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 476 | |
| 477 | if framework == "tf": |
| 478 | try: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 479 | tf_result = np.load(test_path / test_desc["tf_result_npy_filename"]) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 480 | except KeyError: |
| 481 | assert 0, "fail to load tf result numpy" |
| 482 | elif framework == "tflite": |
| 483 | try: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 484 | tf_result = np.load(test_path / test_desc["tflite_result_npy_filename"]) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 485 | except KeyError: |
| 486 | assert 0, "fail to load tflite result numpy" |
| 487 | |
Luke Hutton | 261b7b6 | 2023-01-10 14:50:31 +0000 | [diff] [blame] | 488 | # TOSA has no notion of complex datatypes, it represents complex values using two |
| 489 | # fp32 output tensors representing real and imaginary values. When legalizing |
| 490 | # complex operations from frameworks, these two output tensors are combined into |
| 491 | # a single tensor of shape [?, ..., ?, 2] whereby each inner pair of values |
| 492 | # represents the real and imaginary parts of a complex value. This is completed |
| 493 | # by inserting reshape and concatenate TOSA operations during the legalization to |
| 494 | # maintain a one-to-one correspondance with framework outputs, thus simplifying |
| 495 | # legalization. Here tf_result should also match this format before being |
| 496 | # compared to the ref model output. |
| 497 | if tf_result.dtype == np.complex64: |
| 498 | ifm_shape = tf_result.shape + (2,) |
| 499 | tf_result = tf_result.view(np.float32) |
| 500 | tf_result = tf_result.reshape(ifm_shape) |
| 501 | |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 502 | # Generate test descriptor per flatbuffer generation |
| 503 | # Input .npy will be shared across different frameworks |
| 504 | # Output .npy will be generated in its corresponding flatbuffer |
| 505 | reference_runner_ifm_file = [ |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 506 | str(Path("..") / ifm_file) for ifm_file in test_desc["ifm_file"] |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 507 | ] |
| 508 | |
| 509 | # Check if there's any operator in output graph. |
| 510 | empty_graph = True |
| 511 | with open(tosa_mlir_filename, "r") as f: |
| 512 | for line in f: |
TatWai Chong | e024748 | 2023-06-01 15:02:24 -0700 | [diff] [blame^] | 513 | # TOSA assembly instructions all start with `tosa.` |
| 514 | if re.search(r"tosa\.", line): |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 515 | empty_graph = False |
| 516 | |
| 517 | break |
| 518 | |
| 519 | # Fast-forward input tensor to output tensor if TOSA graph is empty. |
| 520 | if empty_graph: |
| 521 | reference_runner_ofm_name = reference_runner_ifm_name |
| 522 | else: |
| 523 | reference_runner_ofm_name = ["TosaOutput_0"] |
| 524 | |
| 525 | write_reference_runner_json( |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 526 | filename=str(test_path / flatbuffer_dir / "desc.json"), |
| 527 | tosa_filename=f"{test_name}.tosa", |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 528 | ifm_name=reference_runner_ifm_name, |
| 529 | ifm_file=reference_runner_ifm_file, |
| 530 | ofm_name=reference_runner_ofm_name, |
| 531 | ofm_file=["ref_model_output_0.npy"], |
| 532 | ) |
| 533 | |
| 534 | ref_model_cmd = [ |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 535 | str(args.tools_base_dir / "build" / "reference_model" / "tosa_reference_model"), |
| 536 | f"--test_desc={test_path / flatbuffer_dir / 'desc.json'}", |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 537 | ] |
| 538 | |
| 539 | if args.debug_ref_model: |
Eric Kunze | 286f834 | 2022-06-22 11:30:23 -0700 | [diff] [blame] | 540 | ref_model_cmd.extend(["-D ALL", "-l high"]) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 541 | |
Tai Ly | a4d748b | 2023-03-28 22:06:56 +0000 | [diff] [blame] | 542 | if args.precise_mode: |
| 543 | ref_model_cmd.extend(["--precise_mode=1"]) |
| 544 | |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 545 | if args.valgrind: |
| 546 | ref_model_cmd = [ |
| 547 | "valgrind", |
| 548 | "--show-leak-kinds=all", |
| 549 | "--log-fd=1", |
| 550 | "-q", |
| 551 | ] + ref_model_cmd |
| 552 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 553 | ref_model_cmd = ref_model_cmd + [f"--tosa_level={args.tosa_level}"] |
Jerry Ge | a793f46 | 2023-04-11 00:05:02 +0000 | [diff] [blame] | 554 | |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 555 | # Clean out any ref_model result first |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 556 | for f in (test_path / flatbuffer_dir).glob("ref_model_*.npy"): |
| 557 | f.unlink() |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 558 | |
Jared Smolens | b7af461 | 2022-03-21 19:41:52 -0700 | [diff] [blame] | 559 | if args.no_ref: |
| 560 | return (TestResult.PASS, 0.0, msg) |
| 561 | |
| 562 | try: |
| 563 | ref_model_stdout, ref_model_stderr = run_sh_command( |
| 564 | ref_model_cmd, args.verbose, True |
| 565 | ) |
| 566 | ref_model_rc = parse_reference_model_output(ref_model_stdout, ref_model_stderr) |
| 567 | if ref_model_rc != TestResult.PASS: |
| 568 | return (ref_model_rc, 0.0, "") |
| 569 | except Exception as e: |
| 570 | ref_model_rc = parse_reference_model_output("", str(e)) |
| 571 | if ref_model_rc != TestResult.PASS: |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 572 | print_color( |
| 573 | LogColors.RED, |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 574 | f"Results {TestResultErrorStr[ref_model_rc]} {test_name}: {e}", |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 575 | ) |
Jared Smolens | b7af461 | 2022-03-21 19:41:52 -0700 | [diff] [blame] | 576 | return (ref_model_rc, 0.0, "") |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 577 | print_color(LogColors.RED, f"Results REF_MODEL_RUNTIME_ERROR {test_name}: {e}") |
| 578 | return (TestResult.REF_MODEL_RUNTIME_ERROR, 0.0, e, test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 579 | |
Tai Ly | a4d748b | 2023-03-28 22:06:56 +0000 | [diff] [blame] | 580 | if args.precise_mode == 1 and ( |
| 581 | tf_result.dtype == np.float16 or tf_result.dtype == np.float32 |
| 582 | ): |
| 583 | tf_result = tf_result.astype(np.float64) |
| 584 | elif tf_result.dtype == np.float16: |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 585 | tf_result = tf_result.astype(np.float32) |
| 586 | elif ( |
| 587 | tf_result.dtype == np.uint8 |
| 588 | or tf_result.dtype == np.int8 |
| 589 | or tf_result.dtype == np.int16 |
| 590 | or tf_result.dtype == np.int64 |
| 591 | ): |
| 592 | tf_result = tf_result.astype(np.int32) |
| 593 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 594 | # For now, search for the first output from ref_model |
| 595 | ref_model_result_files = list((test_path / flatbuffer_dir).glob("ref_model_*.npy")) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 596 | ref_model_result = np.load(ref_model_result_files[0]) |
| 597 | |
| 598 | assert ( |
| 599 | tf_result.dtype == ref_model_result.dtype |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 600 | ), f"Numpy type mismatch {tf_result.dtype} != {ref_model_result.dtype} when comparing result" |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 601 | |
| 602 | # Size comparison |
| 603 | # Size = 1 tensors can be equivalently represented as having rank 0 or rank |
| 604 | # >= 0, allow that special case |
| 605 | tf_result = np.squeeze(tf_result) |
| 606 | ref_model_result = np.squeeze(ref_model_result) |
| 607 | |
| 608 | if np.shape(tf_result) != np.shape(ref_model_result): |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 609 | print_color(LogColors.RED, f"Results MISCOMPARE {test_name}") |
| 610 | msg = f"Shapes mismatch: Reference {np.shape(tf_result)} vs {np.shape(ref_model_result)}" |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 611 | print(msg) |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 612 | return (TestResult.MISMATCH, 0.0, msg, test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 613 | |
| 614 | # for quantized test, allow +-(args.quantize_tolerance) error |
| 615 | if ref_model_result.dtype == np.int32: |
| 616 | assert tf_result.dtype == np.int32 |
| 617 | |
| 618 | if np.all(np.absolute(ref_model_result - tf_result) <= args.quantize_tolerance): |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 619 | print_color(LogColors.GREEN, f"Results PASS {test_name}") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 620 | else: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 621 | print_color(LogColors.RED, f"Results MISCOMPARE {test_name}") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 622 | |
| 623 | tolerance = args.quantize_tolerance + 1 |
| 624 | while not np.all( |
| 625 | np.absolute(ref_model_result - tf_result) <= args.quantize_tolerance |
| 626 | ): |
| 627 | tolerance = tolerance + 1 |
| 628 | if tolerance >= 10: |
| 629 | break |
| 630 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 631 | msg = f"Result is within {tolerance} {test_path}" |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 632 | print(msg) |
| 633 | |
| 634 | np.set_printoptions(threshold=128) |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 635 | print(f"tf_result: {tf_result.shape}\n") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 636 | print(tf_result) |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 637 | print(f"ref_model_result: {ref_model_result.shape}\n") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 638 | print(ref_model_result) |
| 639 | # print(tf_result - ref_model_result) |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 640 | return (TestResult.MISMATCH, tolerance, msg, test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 641 | else: |
| 642 | if np.allclose( |
| 643 | ref_model_result, tf_result, atol=args.tolerance, equal_nan=True |
| 644 | ): |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 645 | print_color(LogColors.GREEN, f"Results PASS {test_name}") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 646 | else: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 647 | print_color(LogColors.RED, f"Results MISCOMPARE {test_name}") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 648 | |
| 649 | # Many of these tests would match with a reasonable looser tolerence. |
| 650 | # Determine what would have worked. |
| 651 | tolerance = args.tolerance * 10.0 |
| 652 | while not np.allclose( |
| 653 | ref_model_result, tf_result, atol=tolerance, equal_nan=True |
| 654 | ): |
| 655 | tolerance = tolerance * 10.0 |
| 656 | if tolerance > 1.0e10: |
| 657 | tolerance = math.inf |
| 658 | break |
| 659 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 660 | msg = f"Result is within {tolerance:.0e} {test_name}" |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 661 | print(msg) |
| 662 | |
| 663 | np.set_printoptions(precision=4, threshold=128) |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 664 | print(f"tf_result: {tf_result.shape}\n") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 665 | print(tf_result) |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 666 | print(f"ref_model_result: {ref_model_result.shape}\n") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 667 | print(ref_model_result) |
| 668 | # print(tf_result - ref_model_result) |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 669 | return (TestResult.MISMATCH, tolerance, msg, test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 670 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 671 | return (TestResult.PASS, args.tolerance, msg, test_name) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 672 | |
| 673 | |
| 674 | def worker_thread(task_queue, args, result_queue): |
| 675 | while True: |
| 676 | try: |
| 677 | (test, framework) = task_queue.get(block=False) |
| 678 | except queue.Empty: |
| 679 | break |
| 680 | |
| 681 | if test is None: |
| 682 | break |
| 683 | |
| 684 | msg = "" |
| 685 | start_time = datetime.now() |
| 686 | try: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 687 | (rc, tolerance, msg, test_name) = run_test(args, test, framework) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 688 | except Exception as e: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 689 | print(f"Internal regression error: {e}") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 690 | print( |
| 691 | "".join( |
| 692 | traceback.format_exception( |
| 693 | etype=type(e), value=e, tb=e.__traceback__ |
| 694 | ) |
| 695 | ) |
| 696 | ) |
| 697 | rc = TestResult.INTERNAL_ERROR |
| 698 | tolerance = 0.0 |
| 699 | |
| 700 | end_time = datetime.now() |
| 701 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 702 | result_queue.put( |
| 703 | (test, framework, rc, tolerance, msg, end_time - start_time, test_name) |
| 704 | ) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 705 | task_queue.task_done() |
| 706 | |
| 707 | return True |
| 708 | |
| 709 | |
| 710 | def getTestsInDir(directory): |
| 711 | # Recursively find any tests in this directory |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 712 | if (directory / "test.json").is_file(): |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 713 | return [directory] |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 714 | elif directory.is_dir(): |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 715 | test_list = [] |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 716 | for d in directory.glob("*"): |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 717 | test_list.extend(getTestsInDir(d)) |
| 718 | return test_list |
| 719 | else: |
| 720 | return [] |
| 721 | |
| 722 | |
| 723 | def main(): |
| 724 | args = parse_args() |
| 725 | |
| 726 | set_print_in_color(not args.no_color) |
| 727 | |
| 728 | if args.output_file: |
| 729 | set_print_in_color(False) |
| 730 | sys.stdout = open(args.output_file, "w") |
| 731 | |
| 732 | # Disable TF info messages |
| 733 | os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" |
| 734 | |
| 735 | task_queue = queue.Queue() |
| 736 | result_queue = queue.Queue() |
| 737 | |
| 738 | threads = [] |
| 739 | |
| 740 | # Result counters for each of the TestResult return codes |
| 741 | results = [0] * len(TestResult) |
| 742 | |
| 743 | for tdir in args.test: |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 744 | if args.recursive_tests: |
| 745 | tdirList = getTestsInDir(tdir) |
| 746 | else: |
| 747 | tdirList = [tdir] |
| 748 | |
| 749 | for t in tdirList: |
| 750 | for f in args.framework: |
| 751 | task_queue.put((t, f)) |
| 752 | |
| 753 | for i in range(args.jobs): |
| 754 | t = threading.Thread( |
| 755 | target=worker_thread, args=(task_queue, args, result_queue) |
| 756 | ) |
| 757 | t.setDaemon(True) |
| 758 | t.start() |
| 759 | threads.append(t) |
| 760 | |
| 761 | # Run until queue is empty |
| 762 | task_queue.join() |
| 763 | |
| 764 | print_color(LogColors.BOLD_WHITE, "Result summary") |
| 765 | |
| 766 | result_list = [] |
| 767 | while True: |
| 768 | try: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 769 | test, framework, rc, tol, msg, time_delta, test_name = result_queue.get( |
| 770 | block=False |
| 771 | ) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 772 | except queue.Empty: |
| 773 | break |
| 774 | |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 775 | result_list.append((test, framework, rc, tol, msg, time_delta, test_name)) |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 776 | results[rc] = results[rc] + 1 |
| 777 | |
| 778 | xunit_result = xunit_results() |
| 779 | xunit_suite = xunit_result.create_suite(args.xunit_classname_prefix) |
| 780 | |
| 781 | # Sort by test name |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 782 | for test, framework, rc, tol, err_msg, time_delta, test_name in sorted( |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 783 | result_list, key=lambda tup: tup[0] |
| 784 | ): |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 785 | class_name = f"{args.xunit_classname_prefix}.{framework}" |
| 786 | |
| 787 | xt = xunit_test(test_name, class_name) |
| 788 | |
| 789 | msg = TestResultErrorStr[rc] |
| 790 | |
| 791 | xt.time = str( |
| 792 | float(time_delta.seconds) + (float(time_delta.microseconds) * 1e-6) |
| 793 | ) |
| 794 | |
| 795 | if len(msg) > 0: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 796 | print(f"{msg} on {framework} {test}") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 797 | |
| 798 | # Add any more verbose messaging for the xml log |
| 799 | if err_msg: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 800 | msg = f"{msg} {err_msg}" |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 801 | |
| 802 | if rc == TestResult.PASS: |
| 803 | pass |
| 804 | elif rc == TestResult.SKIPPED: |
| 805 | xt.skipped() |
| 806 | else: |
| 807 | xt.failed(msg) |
| 808 | |
| 809 | xunit_suite.tests.append(xt) |
| 810 | |
| 811 | result_queue.task_done() |
| 812 | |
| 813 | xunit_result.write_results(args.xunit_file) |
| 814 | |
| 815 | print("Totals: ", end="") |
| 816 | for result in TestResult: |
Eric Kunze | 3403ded | 2023-07-28 17:23:48 +0000 | [diff] [blame] | 817 | print(f"{results[result]} {result.name.lower()}, ", end="") |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame] | 818 | print() |
| 819 | |
| 820 | if not args.regression_mode and ( |
| 821 | results[TestResult.COMPILER_ERROR] > 0 |
| 822 | or results[TestResult.REF_MODEL_ERROR] > 0 |
| 823 | or results[TestResult.MISMATCH] > 0 |
| 824 | ): |
| 825 | return 1 |
| 826 | |
| 827 | return 0 |
| 828 | |
| 829 | |
| 830 | if __name__ == "__main__": |
| 831 | exit(main()) |