Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 1 | """Tests for tosa_reference_model.""" |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 2 | # Copyright (c) 2022-2023, ARM Limited. |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | import json |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 5 | import re |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 6 | from pathlib import Path |
| 7 | from shutil import rmtree |
| 8 | |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 9 | import conformance.model_files as cmf |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 10 | import numpy as np |
| 11 | import pytest |
| 12 | from checker.tosa_result_checker import test_check as tosa_check |
| 13 | from checker.tosa_result_checker import TestResult as TosaResult |
| 14 | from generator.tosa_verif_build_tests import main as tosa_builder |
| 15 | from runner.run_command import run_sh_command |
| 16 | from runner.run_command import RunShCommandError |
| 17 | |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 18 | # Note: Must rename imports (like test_check) so that pytest doesn't assume its a test function/class |
| 19 | |
| 20 | # Location of reference model binaries |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 21 | REF_MODEL_DIR = Path(__file__).resolve().parents[2] |
| 22 | REF_MODEL_EXE_PATH = cmf.find_tosa_file( |
| 23 | cmf.TosaFileType.REF_MODEL, REF_MODEL_DIR, False |
| 24 | ) |
| 25 | GENERATE_LIB_PATH = cmf.find_tosa_file( |
| 26 | cmf.TosaFileType.GENERATE_LIBRARY, REF_MODEL_EXE_PATH |
| 27 | ) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 28 | |
| 29 | # Set this to False if you want ot preserve the test directories after running |
| 30 | CLEAN_UP_TESTS = True |
| 31 | |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 32 | # Default tensor shape information |
| 33 | SHAPE_LIST = ["10", "5"] |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 34 | SHAPE_DIMS = len(SHAPE_LIST) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 35 | SHAPE_ARG = ",".join(SHAPE_LIST) |
| 36 | SHAPE_OUT = "x".join(SHAPE_LIST) |
| 37 | |
| 38 | # Output file information |
| 39 | OUTPUT_DIR_PREFIX = "_pytest_vtest" |
| 40 | OUTPUT_OFM_FILE = "result_refmodel_pytest.npy" |
| 41 | OUTPUT_RESULT_FILE = "result_numpy_pytest.npy" |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 42 | OUTPUT_CONST_GLOB = "const-*.npy" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 43 | |
| 44 | TEST_DESC_FILENAME = "desc.json" |
Jerry Ge | 0bd4ec8 | 2023-05-01 18:36:43 +0000 | [diff] [blame] | 45 | TOSA_LEVEL = "EIGHTK" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 46 | |
| 47 | # Conversion from refmodel type into the type abbreviation used in the test output |
| 48 | REF_MODEL_TYPE_TO_OUT = { |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 49 | "bool": "b", |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 50 | "int8": "i8", |
| 51 | "uint8": "u8", |
| 52 | "int16": "i16", |
| 53 | "int32": "i32", |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 54 | "fp32": "f32", |
Jeremy Johnson | 93d4390 | 2022-09-27 12:26:14 +0100 | [diff] [blame] | 55 | "fp16": "f16", |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 56 | "bf16": "bf16", |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 57 | } |
| 58 | |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 59 | # NOTE: These tests are marked as POST COMMIT |
| 60 | # To run them, please build the reference_model in a local "build" directory |
| 61 | # (as per the README) and run them using: pytest -m "postcommit" |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 62 | |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 63 | |
| 64 | @pytest.mark.postcommit |
| 65 | def test_refmodel_built(): |
| 66 | """First test to check the reference model has been built.""" |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 67 | assert REF_MODEL_EXE_PATH.is_file() |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 68 | |
| 69 | |
| 70 | class BuildTosaTest: |
| 71 | """Wrapper for managing lifecycle of TOSA unit tests.""" |
| 72 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 73 | def __init__(self, op_name, ref_model_type, num_expected_tests): |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 74 | self.op_name = op_name |
| 75 | self.ref_model_type = ref_model_type |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 76 | self.num_expected_tests = num_expected_tests |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 77 | self.output_dir = None |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 78 | self.test_dirs = None |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 79 | |
| 80 | def create_test(self): |
| 81 | """Helper to generate a TOSA unit test.""" |
| 82 | if self.output_dir is not None: |
| 83 | # Already created |
| 84 | return self.test_dir |
| 85 | |
| 86 | self.output_dir = ( |
| 87 | Path(__file__).parent |
| 88 | / f"{OUTPUT_DIR_PREFIX}_{self.op_name}_{self.ref_model_type}" |
| 89 | ) |
| 90 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 91 | # Generate tests without any zero-point |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 92 | build_args = [ |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 93 | "--generate-lib-path", |
| 94 | str(GENERATE_LIB_PATH), |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 95 | "--filter", |
| 96 | self.op_name, |
| 97 | "--target-shape", |
| 98 | SHAPE_ARG, |
| 99 | "--target-dtype", |
| 100 | self.ref_model_type, |
| 101 | "--zero-point", |
| 102 | "0", |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 103 | "--num-const-inputs-concat", |
| 104 | "1", |
| 105 | "--dump-const-tensors", |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 106 | "-o", |
| 107 | str(self.output_dir), |
| 108 | ] |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 109 | print(f"### Building tests: tosa_verif_build_tests {' '.join(build_args)}") |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 110 | tosa_builder(build_args) |
| 111 | |
| 112 | # Find the created test |
| 113 | test_dir = self.output_dir / self.op_name |
| 114 | # Can't assume exact name due to broadcasting and other changes to shape |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 115 | test_glob = f"{self.op_name}_*_{REF_MODEL_TYPE_TO_OUT[self.ref_model_type]}*" |
| 116 | test_dirs = sorted(test_dir.glob(test_glob)) |
| 117 | assert len(test_dirs) == self.num_expected_tests |
| 118 | for test_dir in test_dirs: |
| 119 | assert test_dir.is_dir() |
| 120 | self.test_dirs = test_dirs |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 121 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 122 | return self.test_dirs |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 123 | |
| 124 | def remove_test(self): |
| 125 | if self.output_dir is not None and self.output_dir.is_dir(): |
| 126 | # Delete directory |
| 127 | test_tree = self.output_dir.resolve() |
| 128 | if CLEAN_UP_TESTS: |
| 129 | print(f"Deleting {test_tree}") |
| 130 | rmtree(str(test_tree)) |
| 131 | self.output_dir = None |
| 132 | else: |
| 133 | print(f"Skipped clean up of {test_tree}") |
| 134 | |
| 135 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 136 | # Tests - op_name, ref_model_type, num_expected_tests |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 137 | TEST_PARAMS = [ |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 138 | ("add", "int32", 1), |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 139 | ("add", "fp32", 1), |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 140 | ("abs", "int32", 1), |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 141 | ("abs", "fp32", 1), |
Jeremy Johnson | 93d4390 | 2022-09-27 12:26:14 +0100 | [diff] [blame] | 142 | ("abs", "fp16", 1), |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 143 | ("abs", "bf16", 1), |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 144 | ("negate", "int8", 1), |
| 145 | ("negate", "int16", 1), |
| 146 | ("negate", "int32", 1), |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 147 | ("negate", "fp32", 1), |
Jeremy Johnson | 93d4390 | 2022-09-27 12:26:14 +0100 | [diff] [blame] | 148 | ("negate", "fp16", 1), |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 149 | ("negate", "bf16", 1), |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 150 | # One test per axis (shape dimensions) |
| 151 | ("concat", "bool", SHAPE_DIMS), |
| 152 | ("concat", "int8", SHAPE_DIMS), |
| 153 | ("concat", "int16", SHAPE_DIMS), |
| 154 | ("concat", "int32", SHAPE_DIMS), |
Jeremy Johnson | bc2a3db | 2022-09-27 13:50:00 +0100 | [diff] [blame] | 155 | ("concat", "fp32", SHAPE_DIMS), |
Jeremy Johnson | 93d4390 | 2022-09-27 12:26:14 +0100 | [diff] [blame] | 156 | ("concat", "fp16", SHAPE_DIMS), |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 157 | ("concat", "bf16", SHAPE_DIMS), |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 158 | ] |
| 159 | |
| 160 | |
| 161 | def id_2_name(id): |
| 162 | """Convert test id to name - otherwise it will be tosaTestN.""" |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 163 | op_name, ref_model_type, _ = id |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 164 | return f"{op_name}-{ref_model_type}" |
| 165 | |
| 166 | |
| 167 | @pytest.fixture(params=TEST_PARAMS, ids=id_2_name) |
| 168 | def tosaTest(request): |
| 169 | """Fixture to generate the required test params and clean up.""" |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 170 | op_name, ref_model_type, num_expected_tests = request.param |
| 171 | tst = BuildTosaTest(op_name, ref_model_type, num_expected_tests) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 172 | yield tst |
| 173 | tst.remove_test() |
| 174 | |
| 175 | |
| 176 | @pytest.mark.postcommit |
| 177 | def test_refmodel_simple_op(tosaTest): |
| 178 | """Operator testing versus Numpy.""" |
| 179 | op_name = tosaTest.op_name |
| 180 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 181 | # Generate TOSA test(s) (mostly should be single test) |
| 182 | test_dirs = tosaTest.create_test() |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 183 | |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 184 | # Indicate miscellaneous checks to run in tosa_check |
| 185 | misc_checks = [] |
| 186 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 187 | for test_dir in test_dirs: |
| 188 | # Run ref model |
| 189 | desc_file = test_dir / TEST_DESC_FILENAME |
| 190 | assert desc_file.is_file() |
| 191 | refmodel_cmd = [ |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 192 | str(REF_MODEL_EXE_PATH), |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 193 | "--test_desc", |
| 194 | str(desc_file), |
| 195 | "--ofm_file", |
| 196 | OUTPUT_OFM_FILE, |
Jerry Ge | 0bd4ec8 | 2023-05-01 18:36:43 +0000 | [diff] [blame] | 197 | "--tosa_level", |
| 198 | TOSA_LEVEL, |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 199 | ] |
| 200 | try: |
| 201 | run_sh_command(refmodel_cmd, verbose=True, capture_output=True) |
| 202 | except RunShCommandError as err: |
| 203 | assert False, f"Unexpected exception {err}" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 204 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 205 | # Find output |
| 206 | ofm_file = test_dir / OUTPUT_OFM_FILE |
| 207 | assert ofm_file.is_file() |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 208 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 209 | # Load inputs for Numpy |
| 210 | with desc_file.open("r") as fp: |
| 211 | test_desc = json.load(fp) |
| 212 | tensors = [] |
| 213 | assert "ifm_file" in test_desc |
| 214 | for input_name in test_desc["ifm_file"]: |
| 215 | input_file = test_dir / input_name |
| 216 | assert input_file.is_file() |
| 217 | tensors.append(np.load(str(input_file))) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 218 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 219 | # Load constants for Numpy |
| 220 | const_files = sorted(test_dir.glob(OUTPUT_CONST_GLOB)) |
| 221 | consts = [] |
| 222 | for const_file in const_files: |
| 223 | assert const_file.is_file() |
| 224 | consts.append(np.load(str(const_file))) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 225 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 226 | # Perform Numpy operation |
| 227 | if op_name == "abs": |
| 228 | assert len(tensors) == 1 |
| 229 | result = np.abs(tensors[0]) |
| 230 | elif op_name == "add": |
| 231 | assert len(tensors) == 2 |
| 232 | result = np.add(tensors[0], tensors[1]) |
| 233 | elif op_name == "concat": |
| 234 | assert len(consts) == 1 |
| 235 | # Get axis from test directory name |
| 236 | match = re.search(r"axis([0-9]+)", test_dir.name) |
| 237 | assert match is not None |
| 238 | axis = int(match.group(1)) |
| 239 | result = np.concatenate((*tensors, consts[0]), axis=axis) |
| 240 | elif op_name == "negate": |
| 241 | assert len(tensors) == 1 |
| 242 | result = np.negative(tensors[0]) |
| 243 | else: |
| 244 | assert False, f"Unknown operation {op_name}" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 245 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 246 | # Save Numpy result |
| 247 | result_file = test_dir / OUTPUT_RESULT_FILE |
| 248 | np.save(str(result_file), result) |
| 249 | assert result_file.is_file() |
| 250 | |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 251 | # Ensure valid bf16 |
| 252 | if tosaTest.ref_model_type == "bf16": |
| 253 | misc_checks.append("bf16") |
| 254 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 255 | # Check Numpy result versus refmodel |
| 256 | check_result, tolerance, msg = tosa_check( |
Jeremy Johnson | e4b08ff | 2022-09-15 10:38:17 +0100 | [diff] [blame] | 257 | result_file, |
| 258 | ofm_file, |
James Ward | 24dbc42 | 2022-10-19 12:20:31 +0100 | [diff] [blame] | 259 | test_name=test_dir.name, |
| 260 | misc_checks=misc_checks, |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 261 | ) |
| 262 | assert check_result == TosaResult.PASS |