Build compliance verifier library with a python interface.
Build steps to create tosa_reference_verify_lib.so
New python class to handle loading and calling verifier with pytests.
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Id42c71c398ed7364742b4a039977412ef3081c4a
diff --git a/verif/tests/test_tosa_verifier.py b/verif/tests/test_tosa_verifier.py
new file mode 100644
index 0000000..9524158
--- /dev/null
+++ b/verif/tests/test_tosa_verifier.py
@@ -0,0 +1,76 @@
+"""Tests for the python interface to the verifier library."""
+# Copyright (c) 2023, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+from pathlib import Path
+
+import numpy as np
+import pytest
+from checker.verifier import VerifierError
+from checker.verifier import VerifierLibrary
+
+# NOTE: These tests are set to POST COMMIT - so will only run on the CI
+
+# Location of reference model binaries
+REF_MODEL_BUILD_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model"
+VERIFIER_LIB = "libtosa_reference_verify_lib.so"
+VERIFIER_LIB_PATH = REF_MODEL_BUILD_PATH / VERIFIER_LIB
+
+
+@pytest.mark.postcommit
+def test_verifier_lib_built():
+ """First test to check the library has been built."""
+ assert VERIFIER_LIB_PATH.is_file()
+
+
+@pytest.mark.postcommit
+def test_checker_verifier_load_fail():
+ with pytest.raises(VerifierError) as excinfo:
+ VerifierLibrary(Path("/place-that-does-not-exist"))
+ assert str(excinfo.value).startswith(f"Could not find {VERIFIER_LIB}")
+
+
+@pytest.mark.postcommit
+def test_checker_verifier_load():
+ vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent)
+ assert vlib
+
+
+JSON_COMPLIANCE_DOT_PRODUCT = {
+ "version": "0.1",
+ "tensors": {
+ "output1": {
+ "mode": "DOT_PRODUCT",
+ "dot_product_info": {"ks": 1000, "s": 0, "data_type": "FP32"},
+ }
+ },
+}
+
+
+@pytest.mark.postcommit
+def test_checker_verifier_dot_product_check():
+ vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent)
+ assert vlib
+
+ imp_arr = np.zeros((10, 10, 10), dtype=np.float32)
+ ref_arr = np.zeros((10, 10, 10), dtype=np.float64)
+ bnd_arr = np.zeros((10, 10, 10), dtype=np.float64)
+
+ json_config = JSON_COMPLIANCE_DOT_PRODUCT
+
+ ret = vlib.verify_data("output1", json_config, imp_arr, ref_arr, bnd_arr)
+ assert ret
+
+
+@pytest.mark.postcommit
+def test_checker_verifier_dot_product_check_fail():
+ vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent)
+ assert vlib
+
+ imp_arr = np.zeros((10, 10, 10), dtype=np.float32)
+ ref_arr = np.ones((10, 10, 10), dtype=np.float64)
+ bnd_arr = np.zeros((10, 10, 10), dtype=np.float64)
+
+ json_config = JSON_COMPLIANCE_DOT_PRODUCT
+
+ ret = vlib.verify_data("output1", json_config, imp_arr, ref_arr, bnd_arr)
+ assert not ret