Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 1 | """Tests for tosa_reference_model.""" |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame^] | 2 | # Copyright (c) 2022-2023, ARM Limited. |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | import json |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 5 | import re |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 6 | from pathlib import Path |
| 7 | from shutil import rmtree |
| 8 | |
| 9 | import numpy as np |
| 10 | import pytest |
| 11 | from checker.tosa_result_checker import test_check as tosa_check |
| 12 | from checker.tosa_result_checker import TestResult as TosaResult |
| 13 | from generator.tosa_verif_build_tests import main as tosa_builder |
| 14 | from runner.run_command import run_sh_command |
| 15 | from runner.run_command import RunShCommandError |
| 16 | |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame^] | 17 | # Note: Must rename imports (like test_check) so that pytest doesn't assume its a test function/class |
| 18 | |
| 19 | # Location of reference model binaries |
| 20 | REF_MODEL_BUILD_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model" |
| 21 | REF_MODEL_EXE = "tosa_reference_model" |
| 22 | REF_MODEL_EXE_PATH = REF_MODEL_BUILD_PATH / REF_MODEL_EXE |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 23 | |
| 24 | # Set this to False if you want ot preserve the test directories after running |
| 25 | CLEAN_UP_TESTS = True |
| 26 | |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 27 | # Default tensor shape information |
| 28 | SHAPE_LIST = ["10", "5"] |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 29 | SHAPE_DIMS = len(SHAPE_LIST) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 30 | SHAPE_ARG = ",".join(SHAPE_LIST) |
| 31 | SHAPE_OUT = "x".join(SHAPE_LIST) |
| 32 | |
| 33 | # Output file information |
| 34 | OUTPUT_DIR_PREFIX = "_pytest_vtest" |
| 35 | OUTPUT_OFM_FILE = "result_refmodel_pytest.npy" |
| 36 | OUTPUT_RESULT_FILE = "result_numpy_pytest.npy" |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 37 | OUTPUT_CONST_GLOB = "const-*.npy" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 38 | |
| 39 | TEST_DESC_FILENAME = "desc.json" |
Jerry Ge | 0bd4ec8 | 2023-05-01 18:36:43 +0000 | [diff] [blame] | 40 | TOSA_LEVEL = "EIGHTK" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 41 | |
| 42 | # Conversion from refmodel type into the type abbreviation used in the test output |
| 43 | REF_MODEL_TYPE_TO_OUT = { |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 44 | "bool": "b", |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 45 | "int8": "i8", |
| 46 | "uint8": "u8", |
| 47 | "int16": "i16", |
| 48 | "int32": "i32", |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 49 | "fp32": "f32", |
Jeremy Johnson | 93d4390 | 2022-09-27 12:26:14 +0100 | [diff] [blame] | 50 | "fp16": "f16", |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 51 | "bf16": "bf16", |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 52 | } |
| 53 | |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame^] | 54 | # NOTE: These tests are set to POST COMMIT - so will only run on the CI |
| 55 | |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 56 | |
| 57 | @pytest.mark.postcommit |
| 58 | def test_refmodel_built(): |
| 59 | """First test to check the reference model has been built.""" |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame^] | 60 | assert REF_MODEL_EXE_PATH.is_file() |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 61 | |
| 62 | |
| 63 | class BuildTosaTest: |
| 64 | """Wrapper for managing lifecycle of TOSA unit tests.""" |
| 65 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 66 | def __init__(self, op_name, ref_model_type, num_expected_tests): |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 67 | self.op_name = op_name |
| 68 | self.ref_model_type = ref_model_type |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 69 | self.num_expected_tests = num_expected_tests |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 70 | self.output_dir = None |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 71 | self.test_dirs = None |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 72 | |
| 73 | def create_test(self): |
| 74 | """Helper to generate a TOSA unit test.""" |
| 75 | if self.output_dir is not None: |
| 76 | # Already created |
| 77 | return self.test_dir |
| 78 | |
| 79 | self.output_dir = ( |
| 80 | Path(__file__).parent |
| 81 | / f"{OUTPUT_DIR_PREFIX}_{self.op_name}_{self.ref_model_type}" |
| 82 | ) |
| 83 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 84 | # Generate tests without any zero-point |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 85 | build_args = [ |
| 86 | "--filter", |
| 87 | self.op_name, |
| 88 | "--target-shape", |
| 89 | SHAPE_ARG, |
| 90 | "--target-dtype", |
| 91 | self.ref_model_type, |
| 92 | "--zero-point", |
| 93 | "0", |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 94 | "--num-const-inputs-concat", |
| 95 | "1", |
| 96 | "--dump-const-tensors", |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 97 | "-o", |
| 98 | str(self.output_dir), |
| 99 | ] |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 100 | print(f"### Building tests: tosa_verif_build_tests {' '.join(build_args)}") |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 101 | tosa_builder(build_args) |
| 102 | |
| 103 | # Find the created test |
| 104 | test_dir = self.output_dir / self.op_name |
| 105 | # Can't assume exact name due to broadcasting and other changes to shape |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 106 | test_glob = f"{self.op_name}_*_{REF_MODEL_TYPE_TO_OUT[self.ref_model_type]}*" |
| 107 | test_dirs = sorted(test_dir.glob(test_glob)) |
| 108 | assert len(test_dirs) == self.num_expected_tests |
| 109 | for test_dir in test_dirs: |
| 110 | assert test_dir.is_dir() |
| 111 | self.test_dirs = test_dirs |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 112 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 113 | return self.test_dirs |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 114 | |
| 115 | def remove_test(self): |
| 116 | if self.output_dir is not None and self.output_dir.is_dir(): |
| 117 | # Delete directory |
| 118 | test_tree = self.output_dir.resolve() |
| 119 | if CLEAN_UP_TESTS: |
| 120 | print(f"Deleting {test_tree}") |
| 121 | rmtree(str(test_tree)) |
| 122 | self.output_dir = None |
| 123 | else: |
| 124 | print(f"Skipped clean up of {test_tree}") |
| 125 | |
| 126 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 127 | # Tests - op_name, ref_model_type, num_expected_tests |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 128 | TEST_PARAMS = [ |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 129 | ("add", "int32", 1), |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 130 | ("add", "fp32", 1), |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 131 | ("abs", "int32", 1), |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 132 | ("abs", "fp32", 1), |
Jeremy Johnson | 93d4390 | 2022-09-27 12:26:14 +0100 | [diff] [blame] | 133 | ("abs", "fp16", 1), |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 134 | ("abs", "bf16", 1), |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 135 | ("negate", "int8", 1), |
| 136 | ("negate", "int16", 1), |
| 137 | ("negate", "int32", 1), |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 138 | ("negate", "fp32", 1), |
Jeremy Johnson | 93d4390 | 2022-09-27 12:26:14 +0100 | [diff] [blame] | 139 | ("negate", "fp16", 1), |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 140 | ("negate", "bf16", 1), |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 141 | # One test per axis (shape dimensions) |
| 142 | ("concat", "bool", SHAPE_DIMS), |
| 143 | ("concat", "int8", SHAPE_DIMS), |
| 144 | ("concat", "int16", SHAPE_DIMS), |
| 145 | ("concat", "int32", SHAPE_DIMS), |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 146 | ("concat", "fp32", SHAPE_DIMS), |
Jeremy Johnson | 93d4390 | 2022-09-27 12:26:14 +0100 | [diff] [blame] | 147 | ("concat", "fp16", SHAPE_DIMS), |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 148 | ("concat", "bf16", SHAPE_DIMS), |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 149 | ] |
| 150 | |
| 151 | |
| 152 | def id_2_name(id): |
| 153 | """Convert test id to name - otherwise it will be tosaTestN.""" |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 154 | op_name, ref_model_type, _ = id |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 155 | return f"{op_name}-{ref_model_type}" |
| 156 | |
| 157 | |
| 158 | @pytest.fixture(params=TEST_PARAMS, ids=id_2_name) |
| 159 | def tosaTest(request): |
| 160 | """Fixture to generate the required test params and clean up.""" |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 161 | op_name, ref_model_type, num_expected_tests = request.param |
| 162 | tst = BuildTosaTest(op_name, ref_model_type, num_expected_tests) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 163 | yield tst |
| 164 | tst.remove_test() |
| 165 | |
| 166 | |
| 167 | @pytest.mark.postcommit |
| 168 | def test_refmodel_simple_op(tosaTest): |
| 169 | """Operator testing versus Numpy.""" |
| 170 | op_name = tosaTest.op_name |
| 171 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 172 | # Generate TOSA test(s) (mostly should be single test) |
| 173 | test_dirs = tosaTest.create_test() |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 174 | |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 175 | # Indicate miscellaneous checks to run in tosa_check |
| 176 | misc_checks = [] |
| 177 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 178 | for test_dir in test_dirs: |
| 179 | # Run ref model |
| 180 | desc_file = test_dir / TEST_DESC_FILENAME |
| 181 | assert desc_file.is_file() |
| 182 | refmodel_cmd = [ |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame^] | 183 | str(REF_MODEL_EXE_PATH), |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 184 | "--test_desc", |
| 185 | str(desc_file), |
| 186 | "--ofm_file", |
| 187 | OUTPUT_OFM_FILE, |
Jerry Ge | 0bd4ec8 | 2023-05-01 18:36:43 +0000 | [diff] [blame] | 188 | "--tosa_level", |
| 189 | TOSA_LEVEL, |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 190 | ] |
| 191 | try: |
| 192 | run_sh_command(refmodel_cmd, verbose=True, capture_output=True) |
| 193 | except RunShCommandError as err: |
| 194 | assert False, f"Unexpected exception {err}" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 195 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 196 | # Find output |
| 197 | ofm_file = test_dir / OUTPUT_OFM_FILE |
| 198 | assert ofm_file.is_file() |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 199 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 200 | # Load inputs for Numpy |
| 201 | with desc_file.open("r") as fp: |
| 202 | test_desc = json.load(fp) |
| 203 | tensors = [] |
| 204 | assert "ifm_file" in test_desc |
| 205 | for input_name in test_desc["ifm_file"]: |
| 206 | input_file = test_dir / input_name |
| 207 | assert input_file.is_file() |
| 208 | tensors.append(np.load(str(input_file))) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 209 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 210 | # Load constants for Numpy |
| 211 | const_files = sorted(test_dir.glob(OUTPUT_CONST_GLOB)) |
| 212 | consts = [] |
| 213 | for const_file in const_files: |
| 214 | assert const_file.is_file() |
| 215 | consts.append(np.load(str(const_file))) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 216 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 217 | # Perform Numpy operation |
| 218 | if op_name == "abs": |
| 219 | assert len(tensors) == 1 |
| 220 | result = np.abs(tensors[0]) |
| 221 | elif op_name == "add": |
| 222 | assert len(tensors) == 2 |
| 223 | result = np.add(tensors[0], tensors[1]) |
| 224 | elif op_name == "concat": |
| 225 | assert len(consts) == 1 |
| 226 | # Get axis from test directory name |
| 227 | match = re.search(r"axis([0-9]+)", test_dir.name) |
| 228 | assert match is not None |
| 229 | axis = int(match.group(1)) |
| 230 | result = np.concatenate((*tensors, consts[0]), axis=axis) |
| 231 | elif op_name == "negate": |
| 232 | assert len(tensors) == 1 |
| 233 | result = np.negative(tensors[0]) |
| 234 | else: |
| 235 | assert False, f"Unknown operation {op_name}" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 236 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 237 | # Save Numpy result |
| 238 | result_file = test_dir / OUTPUT_RESULT_FILE |
| 239 | np.save(str(result_file), result) |
| 240 | assert result_file.is_file() |
| 241 | |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 242 | # Ensure valid bf16 |
| 243 | if tosaTest.ref_model_type == "bf16": |
| 244 | misc_checks.append("bf16") |
| 245 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 246 | # Check Numpy result versus refmodel |
| 247 | check_result, tolerance, msg = tosa_check( |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 248 | result_file, |
| 249 | ofm_file, |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 250 | test_name=test_dir.name, |
| 251 | misc_checks=misc_checks, |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 252 | ) |
| 253 | assert check_result == TosaResult.PASS |