Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | # Copyright (c) 2020, ARM Limited. |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | |
| 18 | import argparse |
| 19 | import sys |
| 20 | import re |
| 21 | import os |
| 22 | import subprocess |
| 23 | import shlex |
| 24 | import json |
| 25 | import glob |
| 26 | import math |
| 27 | import queue |
| 28 | import threading |
| 29 | import traceback |
| 30 | import importlib |
| 31 | |
| 32 | |
| 33 | from enum import IntEnum, Enum, unique |
| 34 | from datetime import datetime |
| 35 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 36 | # Include the ../scripts and ../scripts/xunit directory in PYTHONPATH |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 37 | parent_dir = os.path.dirname(os.path.realpath(__file__)) |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 38 | sys.path.append(os.path.join(parent_dir, "..", "scripts")) |
| 39 | sys.path.append(os.path.join(parent_dir, "..", "scripts", "xunit")) |
| 40 | |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 41 | import xunit |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 42 | |
| 43 | # Include the ../thirdparty/serialization_lib/python directory in PYTHONPATH |
| 44 | parent_dir = os.path.dirname(os.path.realpath(__file__)) |
| 45 | sys.path.append( |
| 46 | os.path.join(parent_dir, "..", "thirdparty", "serialization_lib", "python") |
| 47 | ) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 48 | import tosa |
| 49 | from tosa_test_gen import TosaTestGen |
| 50 | from tosa_test_runner import TosaTestRunner |
| 51 | |
| 52 | no_color_printing = False |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 53 | # from run_tf_unit_test import LogColors, print_color, run_sh_command |
| 54 | |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 55 | |
| 56 | def parseArgs(): |
| 57 | |
| 58 | parser = argparse.ArgumentParser() |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 59 | parser.add_argument( |
Jared Smolens | 7e7fccd | 2021-09-21 11:36:18 -0700 | [diff] [blame] | 60 | "-t", |
| 61 | "--test", |
| 62 | dest="test", |
| 63 | type=str, |
| 64 | nargs="+", |
| 65 | help="Test(s) to run", |
| 66 | required=True, |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 67 | ) |
| 68 | parser.add_argument( |
| 69 | "--seed", |
| 70 | dest="random_seed", |
| 71 | default=42, |
| 72 | type=int, |
| 73 | help="Random seed for test generation", |
| 74 | ) |
| 75 | parser.add_argument( |
| 76 | "--ref-model-path", |
| 77 | dest="ref_model_path", |
| 78 | default="build/reference_model/tosa_reference_model", |
| 79 | type=str, |
| 80 | help="Path to reference model executable", |
| 81 | ) |
| 82 | parser.add_argument( |
| 83 | "--ref-debug", |
| 84 | dest="ref_debug", |
| 85 | default="", |
| 86 | type=str, |
| 87 | help="Reference debug flag (low, med, high)", |
| 88 | ) |
| 89 | parser.add_argument( |
| 90 | "--ref-intermediates", |
| 91 | dest="ref_intermediates", |
| 92 | default=0, |
| 93 | type=int, |
| 94 | help="Reference model dumps intermediate tensors", |
| 95 | ) |
| 96 | parser.add_argument( |
| 97 | "-v", "--verbose", dest="verbose", action="count", help="Verbose operation" |
| 98 | ) |
| 99 | parser.add_argument( |
| 100 | "-j", "--jobs", dest="jobs", type=int, default=1, help="Number of parallel jobs" |
| 101 | ) |
| 102 | parser.add_argument( |
| 103 | "--sut-module", |
| 104 | "-s", |
| 105 | dest="sut_module", |
| 106 | type=str, |
| 107 | nargs="+", |
| 108 | default=["tosa_ref_run"], |
| 109 | help="System under test module to load (derives from TosaTestRunner). May be repeated", |
| 110 | ) |
| 111 | parser.add_argument( |
| 112 | "--sut-module-args", |
| 113 | dest="sut_module_args", |
| 114 | type=str, |
| 115 | nargs="+", |
| 116 | default=[], |
| 117 | help="System under test module arguments. Use sutmodulename:argvalue to pass an argument. May be repeated.", |
| 118 | ) |
| 119 | parser.add_argument( |
| 120 | "--xunit-file", |
| 121 | dest="xunit_file", |
| 122 | type=str, |
| 123 | default="result.xml", |
| 124 | help="XUnit output file", |
| 125 | ) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 126 | |
| 127 | args = parser.parse_args() |
| 128 | |
| 129 | # Autodetect CPU count |
| 130 | if args.jobs <= 0: |
| 131 | args.jobs = os.cpu_count() |
| 132 | |
| 133 | return args |
| 134 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 135 | |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 136 | def workerThread(task_queue, runnerList, args, result_queue): |
| 137 | while True: |
| 138 | try: |
| 139 | test = task_queue.get(block=False) |
| 140 | except queue.Empty: |
| 141 | break |
| 142 | |
| 143 | if test is None: |
| 144 | break |
| 145 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 146 | msg = "" |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 147 | start_time = datetime.now() |
| 148 | try: |
| 149 | |
| 150 | for runnerModule, runnerArgs in runnerList: |
| 151 | if args.verbose: |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 152 | print( |
| 153 | "Running runner {} with test {}".format( |
| 154 | runnerModule.__name__, test |
| 155 | ) |
| 156 | ) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 157 | runner = runnerModule.TosaRefRunner(args, runnerArgs, test) |
| 158 | try: |
| 159 | rc = runner.runModel() |
| 160 | except Exception as e: |
| 161 | rc = TosaTestRunner.Result.INTERNAL_ERROR |
Les Bell | f414b3c | 2021-09-06 11:29:46 +0100 | [diff] [blame] | 162 | print(f"runner.runModel Exception: {e}") |
| 163 | print( |
| 164 | "".join( |
| 165 | traceback.format_exception( |
| 166 | etype=type(e), value=e, tb=e.__traceback__ |
| 167 | ) |
| 168 | ) |
| 169 | ) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 170 | except Exception as e: |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 171 | print("Internal regression error: {}".format(e)) |
| 172 | print( |
| 173 | "".join( |
| 174 | traceback.format_exception( |
| 175 | etype=type(e), value=e, tb=e.__traceback__ |
| 176 | ) |
| 177 | ) |
| 178 | ) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 179 | rc = TosaTestRunner.Result.INTERNAL_ERROR |
| 180 | |
| 181 | end_time = datetime.now() |
| 182 | |
| 183 | result_queue.put((test, rc, msg, end_time - start_time)) |
| 184 | task_queue.task_done() |
| 185 | |
| 186 | return True |
| 187 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 188 | |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 189 | def loadRefModules(args): |
| 190 | # Returns a tuple of (runner_module, [argument list]) |
| 191 | runnerList = [] |
| 192 | for r in args.sut_module: |
| 193 | if args.verbose: |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 194 | print("Loading module {}".format(r)) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 195 | |
| 196 | runner = importlib.import_module(r) |
| 197 | |
| 198 | # Look for arguments associated with this runner |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 199 | runnerArgPrefix = "{}:".format(r) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 200 | runnerArgList = [] |
| 201 | for a in args.sut_module_args: |
| 202 | if a.startswith(runnerArgPrefix): |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 203 | runnerArgList.append(a[len(runnerArgPrefix) :]) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 204 | runnerList.append((runner, runnerArgList)) |
| 205 | |
| 206 | return runnerList |
| 207 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 208 | |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 209 | def main(): |
| 210 | args = parseArgs() |
| 211 | |
| 212 | runnerList = loadRefModules(args) |
| 213 | |
| 214 | threads = [] |
| 215 | taskQueue = queue.Queue() |
| 216 | resultQueue = queue.Queue() |
| 217 | |
| 218 | for t in args.test: |
| 219 | taskQueue.put((t)) |
| 220 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 221 | print("Running {} tests ".format(taskQueue.qsize())) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 222 | |
| 223 | for i in range(args.jobs): |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 224 | t = threading.Thread( |
| 225 | target=workerThread, args=(taskQueue, runnerList, args, resultQueue) |
| 226 | ) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 227 | t.setDaemon(True) |
| 228 | t.start() |
| 229 | threads.append(t) |
| 230 | |
| 231 | taskQueue.join() |
| 232 | |
| 233 | resultList = [] |
| 234 | results = [0] * len(TosaTestRunner.Result) |
| 235 | |
| 236 | while True: |
| 237 | try: |
| 238 | test, rc, msg, time_delta = resultQueue.get(block=False) |
| 239 | except queue.Empty: |
| 240 | break |
| 241 | |
| 242 | resultList.append((test, rc, msg, time_delta)) |
| 243 | results[rc] = results[rc] + 1 |
| 244 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 245 | xunit_result = xunit.xunit_results("Regressions") |
| 246 | xunit_suite = xunit_result.create_suite("Unit tests") |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 247 | |
| 248 | # Sort by test name |
| 249 | for test, rc, msg, time_delta in sorted(resultList, key=lambda tup: tup[0]): |
| 250 | test_name = test |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 251 | xt = xunit.xunit_test(test_name, "reference") |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 252 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 253 | xt.time = str( |
| 254 | float(time_delta.seconds) + (float(time_delta.microseconds) * 1e-6) |
| 255 | ) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 256 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 257 | if ( |
| 258 | rc == TosaTestRunner.Result.EXPECTED_PASS |
| 259 | or rc == TosaTestRunner.Result.EXPECTED_FAILURE |
| 260 | ): |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 261 | if args.verbose: |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 262 | print("{} {}".format(rc.name, test_name)) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 263 | else: |
| 264 | xt.failed(msg) |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 265 | print("{} {}".format(rc.name, test_name)) |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 266 | |
| 267 | xunit_suite.tests.append(xt) |
| 268 | resultQueue.task_done() |
| 269 | |
| 270 | xunit_result.write_results(args.xunit_file) |
| 271 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 272 | print("Totals: ", end="") |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 273 | for result in TosaTestRunner.Result: |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 274 | print("{} {}, ".format(results[result], result.name.lower()), end="") |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 275 | print() |
| 276 | |
| 277 | return 0 |
| 278 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 279 | |
| 280 | if __name__ == "__main__": |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 281 | exit(main()) |