Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 1 | """Tests for the python interface to the verifier library.""" |
| 2 | # Copyright (c) 2023, ARM Limited. |
| 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | from pathlib import Path |
| 5 | |
| 6 | import numpy as np |
| 7 | import pytest |
| 8 | from checker.verifier import VerifierError |
| 9 | from checker.verifier import VerifierLibrary |
| 10 | |
Jeremy Johnson | 65ba809 | 2023-10-09 16:31:13 +0100 | [diff] [blame] | 11 | # NOTE: These tests are marked as POST COMMIT |
| 12 | # To run them, please build the reference_model in a local "build" directory |
| 13 | # (as per the README) and run them using: pytest -m "postcommit" |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 14 | |
| 15 | # Location of reference model binaries |
| 16 | REF_MODEL_BUILD_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model" |
| 17 | VERIFIER_LIB = "libtosa_reference_verify_lib.so" |
| 18 | VERIFIER_LIB_PATH = REF_MODEL_BUILD_PATH / VERIFIER_LIB |
| 19 | |
| 20 | |
| 21 | @pytest.mark.postcommit |
| 22 | def test_verifier_lib_built(): |
| 23 | """First test to check the library has been built.""" |
| 24 | assert VERIFIER_LIB_PATH.is_file() |
| 25 | |
| 26 | |
| 27 | @pytest.mark.postcommit |
| 28 | def test_checker_verifier_load_fail(): |
| 29 | with pytest.raises(VerifierError) as excinfo: |
| 30 | VerifierLibrary(Path("/place-that-does-not-exist")) |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 31 | assert str(excinfo.value).startswith("Could not find verify library") |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 32 | |
| 33 | |
| 34 | @pytest.mark.postcommit |
| 35 | def test_checker_verifier_load(): |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 36 | vlib = VerifierLibrary(VERIFIER_LIB_PATH) |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 37 | assert vlib |
| 38 | |
| 39 | |
| 40 | JSON_COMPLIANCE_DOT_PRODUCT = { |
| 41 | "version": "0.1", |
| 42 | "tensors": { |
| 43 | "output1": { |
| 44 | "mode": "DOT_PRODUCT", |
Jeremy Johnson | bb0935f | 2023-09-14 16:43:48 +0100 | [diff] [blame] | 45 | "data_type": "FP32", |
| 46 | "dot_product_info": {"ks": 1000, "s": 0}, |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 47 | } |
| 48 | }, |
| 49 | } |
| 50 | |
| 51 | |
| 52 | @pytest.mark.postcommit |
| 53 | def test_checker_verifier_dot_product_check(): |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 54 | vlib = VerifierLibrary(VERIFIER_LIB_PATH) |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 55 | assert vlib |
| 56 | |
| 57 | imp_arr = np.zeros((10, 10, 10), dtype=np.float32) |
| 58 | ref_arr = np.zeros((10, 10, 10), dtype=np.float64) |
| 59 | bnd_arr = np.zeros((10, 10, 10), dtype=np.float64) |
| 60 | |
| 61 | json_config = JSON_COMPLIANCE_DOT_PRODUCT |
| 62 | |
| 63 | ret = vlib.verify_data("output1", json_config, imp_arr, ref_arr, bnd_arr) |
| 64 | assert ret |
| 65 | |
| 66 | |
| 67 | @pytest.mark.postcommit |
| 68 | def test_checker_verifier_dot_product_check_fail(): |
Jeremy Johnson | f0348ea | 2023-09-27 16:10:59 +0100 | [diff] [blame] | 69 | vlib = VerifierLibrary(VERIFIER_LIB_PATH) |
Jeremy Johnson | 48df8c7 | 2023-09-12 14:52:34 +0100 | [diff] [blame] | 70 | assert vlib |
| 71 | |
| 72 | imp_arr = np.zeros((10, 10, 10), dtype=np.float32) |
| 73 | ref_arr = np.ones((10, 10, 10), dtype=np.float64) |
| 74 | bnd_arr = np.zeros((10, 10, 10), dtype=np.float64) |
| 75 | |
| 76 | json_config = JSON_COMPLIANCE_DOT_PRODUCT |
| 77 | |
| 78 | ret = vlib.verify_data("output1", json_config, imp_arr, ref_arr, bnd_arr) |
| 79 | assert not ret |