Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 1 | """Template test runner class for running TOSA tests.""" |
| 2 | # Copyright (c) 2020-2022, ARM Limited. |
| 3 | # SPDX-License-Identifier: Apache-2.0 |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 4 | import json |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 5 | from enum import IntEnum |
| 6 | from pathlib import Path |
| 7 | |
| 8 | from checker.tosa_result_checker import LogColors |
| 9 | from checker.tosa_result_checker import print_color |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame^] | 10 | from checker.tosa_result_checker import set_print_in_color |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 11 | from checker.tosa_result_checker import test_check |
| 12 | from json2fbbin import json2fbbin |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 13 | |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 14 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 15 | class TosaTestInvalid(Exception): |
| 16 | """Exception raised for errors loading test description. |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 17 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 18 | Attributes: |
| 19 | path - full path to missing test description file |
| 20 | exception = underlying exception |
| 21 | """ |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 22 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 23 | def __init__(self, path, exception): |
| 24 | """Initialize test not found error.""" |
| 25 | self.path = path |
| 26 | self.exception = exception |
| 27 | self.message = "Invalid test, could not read test description {}: {}".format( |
| 28 | self.path, str(self.exception) |
| 29 | ) |
| 30 | super().__init__(self.message) |
Kevin Cheng | 550ccc5 | 2021-03-03 11:21:43 -0800 | [diff] [blame] | 31 | |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 32 | |
| 33 | class TosaTestRunner: |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 34 | """TOSA Test Runner template class for systems under test.""" |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 35 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 36 | def __init__(self, args, runnerArgs, testDir): |
| 37 | """Initialize and load JSON meta data file.""" |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 38 | self.args = args |
| 39 | self.runnerArgs = runnerArgs |
| 40 | self.testDir = testDir |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 41 | self.testName = Path(self.testDir).name |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 42 | |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame^] | 43 | set_print_in_color(not args.no_color) |
| 44 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 45 | # Check if we want to run binary and if its already converted |
| 46 | descFilePath = Path(testDir, "desc.json") |
| 47 | descBinFilePath = Path(testDir, "desc_binary.json") |
| 48 | if args.binary: |
| 49 | if descBinFilePath.is_file(): |
| 50 | descFilePath = descBinFilePath |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 51 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 52 | try: |
| 53 | # Load the json test file |
| 54 | with open(descFilePath, "r") as fd: |
| 55 | self.testDesc = json.load(fd) |
| 56 | except Exception as e: |
| 57 | raise TosaTestInvalid(str(descFilePath), e) |
| 58 | |
| 59 | # Convert to binary if needed |
| 60 | tosaFilePath = Path(testDir, self.testDesc["tosa_file"]) |
| 61 | if args.binary and tosaFilePath.suffix == ".json": |
| 62 | # Convert tosa JSON to binary |
| 63 | json2fbbin.json_to_fbbin( |
| 64 | Path(args.flatc_path), |
| 65 | Path(args.operator_fbs), |
| 66 | tosaFilePath, |
| 67 | Path(testDir), |
| 68 | ) |
| 69 | # Write new desc_binary file |
| 70 | self.testDesc["tosa_file"] = tosaFilePath.stem + ".tosa" |
| 71 | with open(descBinFilePath, "w") as fd: |
| 72 | json.dump(self.testDesc, fd, indent=2) |
| 73 | descFilePath = descBinFilePath |
| 74 | |
| 75 | # Set location of desc.json (or desc_binary.json) file in use |
| 76 | self.descFile = str(descFilePath) |
| 77 | |
| 78 | def skipTest(self): |
| 79 | """Check if the test is skipped due to test type selection.""" |
| 80 | expectedFailure = self.testDesc["expected_failure"] |
| 81 | if self.args.test_type == "negative" and not expectedFailure: |
| 82 | return True |
| 83 | elif self.args.test_type == "positive" and expectedFailure: |
| 84 | return True |
| 85 | return False |
| 86 | |
| 87 | def runTestGraph(self): |
| 88 | """Override with function that calls system under test.""" |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 89 | pass |
| 90 | |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 91 | def testResult(self, tosaGraphResult, graphMessage=None): |
| 92 | """Work out test result based on graph result and output files.""" |
| 93 | expectedFailure = self.testDesc["expected_failure"] |
| 94 | print_result_line = True |
| 95 | |
| 96 | if tosaGraphResult == TosaTestRunner.TosaGraphResult.TOSA_VALID: |
| 97 | if expectedFailure: |
| 98 | result = TosaTestRunner.Result.UNEXPECTED_PASS |
| 99 | resultMessage = "Expected failure test incorrectly passed" |
| 100 | else: |
| 101 | # Work through all the results produced by the testing, assuming success |
| 102 | # but overriding this with any failures found |
| 103 | result = TosaTestRunner.Result.EXPECTED_PASS |
| 104 | messages = [] |
| 105 | for resultNum, resultFileName in enumerate(self.testDesc["ofm_file"]): |
| 106 | if "expected_result_file" in self.testDesc: |
| 107 | try: |
| 108 | conformanceFile = Path( |
| 109 | self.testDir, |
| 110 | self.testDesc["expected_result_file"][resultNum], |
| 111 | ) |
| 112 | except IndexError: |
| 113 | result = TosaTestRunner.Result.INTERNAL_ERROR |
| 114 | msg = "Internal error: Missing expected_result_file {} in {}".format( |
| 115 | resultNum, self.descFile |
| 116 | ) |
| 117 | messages.append(msg) |
| 118 | print(msg) |
| 119 | break |
| 120 | else: |
| 121 | conformanceFile = None |
| 122 | resultFile = Path(self.testDir, resultFileName) |
| 123 | |
| 124 | if conformanceFile: |
| 125 | print_result_line = False # Checker will print one for us |
| 126 | chkResult, tolerance, msg = test_check( |
| 127 | str(conformanceFile), |
| 128 | str(resultFile), |
| 129 | test_name=self.testName, |
| 130 | ) |
| 131 | # Change EXPECTED_PASS assumption if we have any failures |
| 132 | if chkResult != 0: |
| 133 | result = TosaTestRunner.Result.UNEXPECTED_FAILURE |
| 134 | messages.append(msg) |
| 135 | if self.args.verbose: |
| 136 | print(msg) |
| 137 | else: |
| 138 | # No conformance file to verify, just check results file exists |
| 139 | if not resultFile.is_file(): |
| 140 | result = TosaTestRunner.Result.UNEXPECTED_FAILURE |
| 141 | msg = "Results file is missing: {}".format(resultFile) |
| 142 | messages.append(msg) |
| 143 | print(msg) |
| 144 | |
| 145 | if resultFile.is_file(): |
| 146 | # Move the resultFile to allow subsequent system under |
| 147 | # tests to create them and to test they have been created |
| 148 | resultFile = resultFile.rename( |
| 149 | resultFile.with_suffix( |
| 150 | ".{}{}".format(self.__module__, resultFile.suffix) |
| 151 | ) |
| 152 | ) |
| 153 | |
| 154 | resultMessage = "\n".join(messages) if len(messages) > 0 else None |
| 155 | else: |
| 156 | if ( |
| 157 | expectedFailure |
| 158 | and tosaGraphResult == TosaTestRunner.TosaGraphResult.TOSA_ERROR |
| 159 | ): |
| 160 | result = TosaTestRunner.Result.EXPECTED_FAILURE |
| 161 | resultMessage = None |
| 162 | else: |
| 163 | result = TosaTestRunner.Result.UNEXPECTED_FAILURE |
| 164 | resultMessage = graphMessage |
| 165 | |
| 166 | if print_result_line: |
| 167 | if ( |
| 168 | result == TosaTestRunner.Result.EXPECTED_FAILURE |
| 169 | or result == TosaTestRunner.Result.EXPECTED_PASS |
| 170 | ): |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame^] | 171 | print_color( |
| 172 | LogColors.GREEN, "Result code PASS {}".format(self.testName) |
| 173 | ) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 174 | else: |
Jeremy Johnson | 015c355 | 2022-02-23 12:15:03 +0000 | [diff] [blame^] | 175 | print_color(LogColors.RED, "Result code FAIL {}".format(self.testName)) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 176 | |
| 177 | return result, resultMessage |
| 178 | |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 179 | class Result(IntEnum): |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 180 | """Test result codes.""" |
| 181 | |
Eric Kunze | e5e2676 | 2020-10-13 16:11:07 -0700 | [diff] [blame] | 182 | EXPECTED_PASS = 0 |
| 183 | EXPECTED_FAILURE = 1 |
| 184 | UNEXPECTED_PASS = 2 |
| 185 | UNEXPECTED_FAILURE = 3 |
| 186 | INTERNAL_ERROR = 4 |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 187 | SKIPPED = 5 |
| 188 | |
| 189 | class TosaGraphResult(IntEnum): |
| 190 | """The tosa_graph_result codes.""" |
| 191 | |
| 192 | TOSA_VALID = 0 |
| 193 | TOSA_UNPREDICTABLE = 1 |
| 194 | TOSA_ERROR = 2 |
| 195 | OTHER_ERROR = 3 |