Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame^] | 1 | """TOSA test runner module for a mock System Under Test (SUT).""" |
| 2 | # Copyright (c) 2021, ARM Limited. |
| 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | import os |
| 5 | |
| 6 | from runner.run_command import run_sh_command |
| 7 | from runner.run_command import RunShCommandError |
| 8 | from runner.tosa_test_runner import TosaTestRunner |
| 9 | |
| 10 | |
| 11 | class TosaSUTRunner(TosaTestRunner): |
| 12 | """TOSA mock SUT runner.""" |
| 13 | |
| 14 | def __init__(self, args, runnerArgs, testDir): |
| 15 | """Initialize using the given test details.""" |
| 16 | super().__init__(args, runnerArgs, testDir) |
| 17 | |
| 18 | def runTestGraph(self): |
| 19 | """Run the test on a mock SUT.""" |
| 20 | # Read the command line sut-module-args in form arg=value |
| 21 | # and put them in a dictionary |
| 22 | # Note: On the command line (for this module) they look like: |
| 23 | # tests.tosa_mock_sut_run:arg=value |
| 24 | sutArgs = {} |
| 25 | for runArg in self.runnerArgs: |
| 26 | try: |
| 27 | arg, value = runArg.split("=", 1) |
| 28 | except ValueError: |
| 29 | # Argument without a value - treat it as a flag |
| 30 | arg = runArg |
| 31 | value = True |
| 32 | sutArgs[arg] = value |
| 33 | print(f"MOCK SUT: Runner argument dictionary: {sutArgs}") |
| 34 | |
| 35 | # Useful meta data and arguments |
| 36 | tosaFlatbufferSchema = self.args.operator_fbs |
| 37 | tosaSubgraphFile = self.testDesc["tosa_file"] |
| 38 | tosaTestDirectory = self.testDir |
| 39 | tosaTestDescFile = self.descFile |
| 40 | |
| 41 | # Expected file name for the graph results on valid graph |
| 42 | graphResultFiles = [] |
| 43 | for idx, name in enumerate(self.testDesc["ofm_name"]): |
| 44 | graphResultFiles.append( |
| 45 | "{}:{}".format(name, self.testDesc["ofm_file"][idx]) |
| 46 | ) |
| 47 | |
| 48 | # Build up input "tensor_name":"filename" list |
| 49 | tosaInputTensors = [] |
| 50 | for idx, name in enumerate(self.testDesc["ifm_name"]): |
| 51 | tosaInputTensors.append( |
| 52 | "{}:{}".format(name, self.testDesc["ifm_file"][idx]) |
| 53 | ) |
| 54 | |
| 55 | # Build up command line |
| 56 | cmd = [ |
| 57 | "echo", |
| 58 | f"FBS={tosaFlatbufferSchema}", |
| 59 | f"Path={tosaTestDirectory}", |
| 60 | f"Desc={tosaTestDescFile}", |
| 61 | f"Graph={tosaSubgraphFile}", |
| 62 | "Results={}".format(",".join(graphResultFiles)), |
| 63 | "Inputs={}".format(",".join(tosaInputTensors)), |
| 64 | ] |
| 65 | |
| 66 | # Run test on implementation |
| 67 | graphResult = None |
| 68 | graphMessage = None |
| 69 | try: |
| 70 | stdout, stderr = run_sh_command(cmd, verbose=True, capture_output=True) |
| 71 | except RunShCommandError as e: |
| 72 | # Return codes can be used to indicate graphResult status (see tosa_ref_run.py) |
| 73 | # But in this mock version we just set the result based on sutArgs below |
| 74 | print(f"MOCK SUT: Unexpected error {e.return_code} from command: {e}") |
| 75 | graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR |
| 76 | graphMessage = e.stderr |
| 77 | |
| 78 | # Other mock system testing |
| 79 | if self.args.binary: |
| 80 | # Check that the mock binary conversion has happened |
| 81 | _, ext = os.path.splitext(tosaSubgraphFile) |
| 82 | if ( |
| 83 | os.path.basename(tosaTestDescFile) != "desc_binary.json" |
| 84 | and ext != ".tosa" |
| 85 | ): |
| 86 | graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR |
| 87 | |
| 88 | # Mock up graph result based on passed arguments |
| 89 | if not graphResult: |
| 90 | try: |
| 91 | if sutArgs["graph"] == "valid": |
| 92 | graphResult = TosaTestRunner.TosaGraphResult.TOSA_VALID |
| 93 | # Create dummy output file(s) for passing result checker |
| 94 | for idx, fname in enumerate(self.testDesc["ofm_file"]): |
| 95 | if "num_results" in sutArgs and idx == int( |
| 96 | sutArgs["num_results"] |
| 97 | ): |
| 98 | # Skip writing any more to test results checker |
| 99 | break |
| 100 | print("Created " + fname) |
| 101 | fp = open(os.path.join(tosaTestDirectory, fname), "w") |
| 102 | fp.close() |
| 103 | elif sutArgs["graph"] == "error": |
| 104 | graphResult = TosaTestRunner.TosaGraphResult.TOSA_ERROR |
| 105 | graphMessage = "MOCK SUT: ERROR_IF" |
| 106 | elif sutArgs["graph"] == "unpredictable": |
| 107 | graphResult = TosaTestRunner.TosaGraphResult.TOSA_UNPREDICTABLE |
| 108 | graphMessage = "MOCK SUT: UNPREDICTABLE" |
| 109 | else: |
| 110 | graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR |
| 111 | graphMessage = "MOCK SUT: error from system under test" |
| 112 | except KeyError: |
| 113 | graphMessage = "MOCK SUT: No graph result specified!" |
| 114 | print(graphMessage) |
| 115 | graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR |
| 116 | |
| 117 | # Return graph result and message |
| 118 | return graphResult, graphMessage |