commit e2b5e87804e158cb3e5d06a131c317b3890b87b3
author Jeremy Johnson <jeremy.johnson@arm.com> Thu Sep 14 17:02:09 2023 +0100
committer Jeremy Johnson <jeremy.johnson@arm.com> Mon Oct 02 12:04:44 2023 +0100
tree fd8b5a4d56dfcea4be4e6ced73f2d4d5b2e1d92d
parent bb0935f868a5ab09403cf3628848655b06ac1dec

Support for compliance checking testing

Updated to conformance generator to not generate tests with results for
compliance tests.
Updated test runner to run compliance mode version (precise & abs mode)
of reference model to create test results to use against SUT results.
Updated reference model to enable abs_mode on correct desc.json flags.
Updated test checker to support compliance checking using verifier lib.
Seperated color printing from test checker.

Change-Id: I7e2fbfc6883916caa5d94d4ece122c48bf45f530
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>

verif/conformance/tosa_verif_conformance_generator.py

diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index 564617b..c9a0b3a 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -234,6 +234,14 @@
         )
 
     for test in tests:
+        desc = test / "desc.json"
+        with desc.open("r") as fd:
+            test_desc = json.load(fd)
+        if "meta" in test_desc and "compliance" in test_desc["meta"]:
+            logger.info(
+                f"Skipping generating results for new compliance test - {str(test)}"
+            )
+            continue
         ref_cmd = ref_cmd_base.copy()
         ref_cmd.append(str(test))
         ref_cmds.append(ref_cmd)