Improve conformance backward version support
Turn off ARGMAX rank 6 tests for v0.60.0
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I535b7887a9db099d39ad943e151f870ebc716791
diff --git a/verif/conformance/README.md b/verif/conformance/README.md
index 8fbf4fd..d047802 100644
--- a/verif/conformance/README.md
+++ b/verif/conformance/README.md
@@ -27,6 +27,9 @@
Each generation group is a dictionary that contains:
+* "from_version" - optional version string for when the tests have been introduced in TOSA
+of the form "vM.mm.p" where `M` is the major version, `mm` is the minor version
+and `p` is the patch version
* "no_negative_tests" - optional "true" indicator that no negative tests are relevant/generated
* "negative_dim_range" - optional range of dimensions for negative tests
* "generator_args" - list of argument lists to supply to the `tosa_verif_build_tests` (see that tool for more details)
diff --git a/verif/conformance/tosa_base_profile_ops_info.json b/verif/conformance/tosa_base_profile_ops_info.json
index 772602b..3a8622b 100644
--- a/verif/conformance/tosa_base_profile_ops_info.json
+++ b/verif/conformance/tosa_base_profile_ops_info.json
@@ -179,6 +179,7 @@
]
},
"8k_level": {
+ "from_version": "v0.70.0",
"no_negative_tests": "true",
"generator_args": [
[
@@ -1325,6 +1326,7 @@
],
"generation": {
"standard": {
+ "from_version": "v0.80.0",
"generator_args": [
[
"--target-dtype",
@@ -1347,6 +1349,7 @@
]
},
"8k_level": {
+ "from_version": "v0.80.0",
"no_negative_tests": "true",
"selector": "8k_level",
"generator_args": [
diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json
index 054b9d4..0b6dc79 100644
--- a/verif/conformance/tosa_main_profile_ops_info.json
+++ b/verif/conformance/tosa_main_profile_ops_info.json
@@ -794,6 +794,7 @@
],
"generation": {
"standard": {
+ "from_version": "v0.80.0",
"generator_args": [
[
"--target-dtype",
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index 692e79a..a10e839 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -16,6 +16,7 @@
import logging
import multiprocessing as mp
import os
+import re
import shlex
import shutil
import subprocess
@@ -29,6 +30,7 @@
from convert2conformance.convert2conformance import OUTPUT_TYPE_DEFAULT
from convert2conformance.convert2conformance import OUTPUT_TYPES
from distutils.dir_util import copy_tree
+from serializer.tosa_serializer import TOSA_VERSION
logging.basicConfig()
logger = logging.getLogger("tosa_verif_conformance_generator")
@@ -56,6 +58,7 @@
TEST_VERSION_LATEST = "latest"
TEST_VERSION_V0_60_0 = "v0.60.0"
TEST_VERSIONS = (TEST_VERSION_LATEST, TEST_VERSION_V0_60_0)
+REGEX_VERSION = re.compile(r"v([0-9]+)\.([0-9]+)\.([0-9]+)")
class GenConformanceError(Exception):
@@ -644,6 +647,39 @@
return args
+def in_version(test_version, gen_dict):
+ """Check if the selected test_version is compatible with the tests."""
+
+ def version_string_to_numbers(verstr):
+ # Turn the "vM.mm.pp" string into Major, Minor, Patch versions
+ if verstr == TEST_VERSION_LATEST:
+ return (TOSA_VERSION[0], TOSA_VERSION[1], TOSA_VERSION[2])
+ else:
+ match = re.match(REGEX_VERSION, verstr)
+ if match is None:
+ raise KeyError(f"Invalid version string {verstr}")
+ return (int(v) for v in match.groups())
+
+ if "from_version" in gen_dict:
+ selected_version = version_string_to_numbers(test_version)
+ from_version = version_string_to_numbers(gen_dict["from_version"])
+
+ # Check the Major version is compatible, then Minor, and lastly Patch
+ # Unless the versions match, we can exit early due to obvious precedence
+ for sel, fro in zip(selected_version, from_version):
+ if sel < fro:
+ # From version is later than selected version
+ return False
+ elif sel > fro:
+ # From version is earlier than selected version
+ return True
+ # If we get here, the version numbers match exactly
+ return True
+ else:
+ # No specific version info
+ return True
+
+
def main():
args = parse_args()
@@ -826,10 +862,6 @@
)
continue
- if args.test_version == TEST_VERSION_V0_60_0 and op in ("dim",):
- logger.warning(f"{op} is not in {args.test_version} - skipping")
- continue
-
op_profiles_list = test_params[op]["profile"]
if (
args.profile != PROFILES_ALL
@@ -849,6 +881,13 @@
# Iterate through the generation groups selecting tests from each
for gen_name, gen_dict in test_params[op]["generation"].items():
+
+ if not in_version(args.test_version, gen_dict):
+ logger.warning(
+ f"{op} [{gen_name}] is not in {args.test_version} - skipping"
+ )
+ continue
+
no_neg_tests = (
"no_negative_tests" in gen_dict
and gen_dict["no_negative_tests"] == "true"