Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 1 | """Tests for tosa_reference_model.""" |
| 2 | # Copyright (c) 2022, ARM Limited. |
| 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | import json |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 5 | import re |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 6 | from pathlib import Path |
| 7 | from shutil import rmtree |
| 8 | |
| 9 | import numpy as np |
| 10 | import pytest |
| 11 | from checker.tosa_result_checker import test_check as tosa_check |
| 12 | from checker.tosa_result_checker import TestResult as TosaResult |
| 13 | from generator.tosa_verif_build_tests import main as tosa_builder |
| 14 | from runner.run_command import run_sh_command |
| 15 | from runner.run_command import RunShCommandError |
| 16 | |
| 17 | # Note: Must rename imports so that pytest doesn't assume its a test function/class |
| 18 | |
| 19 | # Set this to False if you want ot preserve the test directories after running |
| 20 | CLEAN_UP_TESTS = True |
| 21 | |
| 22 | # Location of reference model binary |
| 23 | REF_MODEL_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model" |
| 24 | REF_MODEL_EXE = "tosa_reference_model" |
| 25 | REF_MODEL = REF_MODEL_PATH / REF_MODEL_EXE |
| 26 | |
| 27 | # Default tensor shape information |
| 28 | SHAPE_LIST = ["10", "5"] |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 29 | SHAPE_DIMS = len(SHAPE_LIST) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 30 | SHAPE_ARG = ",".join(SHAPE_LIST) |
| 31 | SHAPE_OUT = "x".join(SHAPE_LIST) |
| 32 | |
| 33 | # Output file information |
| 34 | OUTPUT_DIR_PREFIX = "_pytest_vtest" |
| 35 | OUTPUT_OFM_FILE = "result_refmodel_pytest.npy" |
| 36 | OUTPUT_RESULT_FILE = "result_numpy_pytest.npy" |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 37 | OUTPUT_CONST_GLOB = "const-*.npy" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 38 | |
| 39 | TEST_DESC_FILENAME = "desc.json" |
| 40 | |
| 41 | # Conversion from refmodel type into the type abbreviation used in the test output |
| 42 | REF_MODEL_TYPE_TO_OUT = { |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 43 | "bool": "b", |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 44 | "int8": "i8", |
| 45 | "uint8": "u8", |
| 46 | "int16": "i16", |
| 47 | "int32": "i32", |
| 48 | "float": "float", |
| 49 | } |
| 50 | |
| 51 | |
| 52 | @pytest.mark.postcommit |
| 53 | def test_refmodel_built(): |
| 54 | """First test to check the reference model has been built.""" |
| 55 | assert REF_MODEL.is_file() |
| 56 | |
| 57 | |
| 58 | class BuildTosaTest: |
| 59 | """Wrapper for managing lifecycle of TOSA unit tests.""" |
| 60 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 61 | def __init__(self, op_name, ref_model_type, num_expected_tests): |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 62 | self.op_name = op_name |
| 63 | self.ref_model_type = ref_model_type |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 64 | self.num_expected_tests = num_expected_tests |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 65 | self.output_dir = None |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 66 | self.test_dirs = None |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 67 | |
| 68 | def create_test(self): |
| 69 | """Helper to generate a TOSA unit test.""" |
| 70 | if self.output_dir is not None: |
| 71 | # Already created |
| 72 | return self.test_dir |
| 73 | |
| 74 | self.output_dir = ( |
| 75 | Path(__file__).parent |
| 76 | / f"{OUTPUT_DIR_PREFIX}_{self.op_name}_{self.ref_model_type}" |
| 77 | ) |
| 78 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 79 | # Generate tests without any zero-point |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 80 | build_args = [ |
| 81 | "--filter", |
| 82 | self.op_name, |
| 83 | "--target-shape", |
| 84 | SHAPE_ARG, |
| 85 | "--target-dtype", |
| 86 | self.ref_model_type, |
| 87 | "--zero-point", |
| 88 | "0", |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 89 | "--num-const-inputs-concat", |
| 90 | "1", |
| 91 | "--dump-const-tensors", |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 92 | "-o", |
| 93 | str(self.output_dir), |
| 94 | ] |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 95 | print(f"### Building tests: tosa_verif_build_tests {' '.join(build_args)}") |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 96 | tosa_builder(build_args) |
| 97 | |
| 98 | # Find the created test |
| 99 | test_dir = self.output_dir / self.op_name |
| 100 | # Can't assume exact name due to broadcasting and other changes to shape |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 101 | test_glob = f"{self.op_name}_*_{REF_MODEL_TYPE_TO_OUT[self.ref_model_type]}*" |
| 102 | test_dirs = sorted(test_dir.glob(test_glob)) |
| 103 | assert len(test_dirs) == self.num_expected_tests |
| 104 | for test_dir in test_dirs: |
| 105 | assert test_dir.is_dir() |
| 106 | self.test_dirs = test_dirs |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 107 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 108 | return self.test_dirs |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 109 | |
| 110 | def remove_test(self): |
| 111 | if self.output_dir is not None and self.output_dir.is_dir(): |
| 112 | # Delete directory |
| 113 | test_tree = self.output_dir.resolve() |
| 114 | if CLEAN_UP_TESTS: |
| 115 | print(f"Deleting {test_tree}") |
| 116 | rmtree(str(test_tree)) |
| 117 | self.output_dir = None |
| 118 | else: |
| 119 | print(f"Skipped clean up of {test_tree}") |
| 120 | |
| 121 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 122 | # Tests - op_name, ref_model_type, num_expected_tests |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 123 | TEST_PARAMS = [ |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 124 | ("add", "int32", 1), |
| 125 | ("add", "float", 1), |
| 126 | ("abs", "int32", 1), |
| 127 | ("abs", "float", 1), |
| 128 | ("negate", "int8", 1), |
| 129 | ("negate", "int16", 1), |
| 130 | ("negate", "int32", 1), |
| 131 | ("negate", "float", 1), |
| 132 | # One test per axis (shape dimensions) |
| 133 | ("concat", "bool", SHAPE_DIMS), |
| 134 | ("concat", "int8", SHAPE_DIMS), |
| 135 | ("concat", "int16", SHAPE_DIMS), |
| 136 | ("concat", "int32", SHAPE_DIMS), |
| 137 | ("concat", "float", SHAPE_DIMS), |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 138 | ] |
| 139 | |
| 140 | |
| 141 | def id_2_name(id): |
| 142 | """Convert test id to name - otherwise it will be tosaTestN.""" |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 143 | op_name, ref_model_type, _ = id |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 144 | return f"{op_name}-{ref_model_type}" |
| 145 | |
| 146 | |
| 147 | @pytest.fixture(params=TEST_PARAMS, ids=id_2_name) |
| 148 | def tosaTest(request): |
| 149 | """Fixture to generate the required test params and clean up.""" |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 150 | op_name, ref_model_type, num_expected_tests = request.param |
| 151 | tst = BuildTosaTest(op_name, ref_model_type, num_expected_tests) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 152 | yield tst |
| 153 | tst.remove_test() |
| 154 | |
| 155 | |
| 156 | @pytest.mark.postcommit |
| 157 | def test_refmodel_simple_op(tosaTest): |
| 158 | """Operator testing versus Numpy.""" |
| 159 | op_name = tosaTest.op_name |
| 160 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 161 | # Generate TOSA test(s) (mostly should be single test) |
| 162 | test_dirs = tosaTest.create_test() |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 163 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 164 | for test_dir in test_dirs: |
| 165 | # Run ref model |
| 166 | desc_file = test_dir / TEST_DESC_FILENAME |
| 167 | assert desc_file.is_file() |
| 168 | refmodel_cmd = [ |
| 169 | str(REF_MODEL), |
| 170 | "--test_desc", |
| 171 | str(desc_file), |
| 172 | "--ofm_file", |
| 173 | OUTPUT_OFM_FILE, |
| 174 | ] |
| 175 | try: |
| 176 | run_sh_command(refmodel_cmd, verbose=True, capture_output=True) |
| 177 | except RunShCommandError as err: |
| 178 | assert False, f"Unexpected exception {err}" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 179 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 180 | # Find output |
| 181 | ofm_file = test_dir / OUTPUT_OFM_FILE |
| 182 | assert ofm_file.is_file() |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 183 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 184 | # Load inputs for Numpy |
| 185 | with desc_file.open("r") as fp: |
| 186 | test_desc = json.load(fp) |
| 187 | tensors = [] |
| 188 | assert "ifm_file" in test_desc |
| 189 | for input_name in test_desc["ifm_file"]: |
| 190 | input_file = test_dir / input_name |
| 191 | assert input_file.is_file() |
| 192 | tensors.append(np.load(str(input_file))) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 193 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 194 | # Load constants for Numpy |
| 195 | const_files = sorted(test_dir.glob(OUTPUT_CONST_GLOB)) |
| 196 | consts = [] |
| 197 | for const_file in const_files: |
| 198 | assert const_file.is_file() |
| 199 | consts.append(np.load(str(const_file))) |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 200 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 201 | # Perform Numpy operation |
| 202 | if op_name == "abs": |
| 203 | assert len(tensors) == 1 |
| 204 | result = np.abs(tensors[0]) |
| 205 | elif op_name == "add": |
| 206 | assert len(tensors) == 2 |
| 207 | result = np.add(tensors[0], tensors[1]) |
| 208 | elif op_name == "concat": |
| 209 | assert len(consts) == 1 |
| 210 | # Get axis from test directory name |
| 211 | match = re.search(r"axis([0-9]+)", test_dir.name) |
| 212 | assert match is not None |
| 213 | axis = int(match.group(1)) |
| 214 | result = np.concatenate((*tensors, consts[0]), axis=axis) |
| 215 | elif op_name == "negate": |
| 216 | assert len(tensors) == 1 |
| 217 | result = np.negative(tensors[0]) |
| 218 | else: |
| 219 | assert False, f"Unknown operation {op_name}" |
Jeremy Johnson | 0042343 | 2022-09-12 17:27:37 +0100 | [diff] [blame] | 220 | |
Jeremy Johnson | a0848c6 | 2022-09-15 15:01:30 +0100 | [diff] [blame] | 221 | # Save Numpy result |
| 222 | result_file = test_dir / OUTPUT_RESULT_FILE |
| 223 | np.save(str(result_file), result) |
| 224 | assert result_file.is_file() |
| 225 | |
| 226 | # Check Numpy result versus refmodel |
| 227 | check_result, tolerance, msg = tosa_check( |
| 228 | str(result_file), str(ofm_file), test_name=test_dir.name |
| 229 | ) |
| 230 | assert check_result == TosaResult.PASS |