Conformance simple backward version and binary files support
Add --test-version option to tosa_verif_conformance_generator to
select the version for tests to output.
Add --output-type to allow json, binary or both files to be created
during conformance generation.
Fix passing schema_path to test runner.
Add explicit verify lib path arg to test runner.
Change-Id: I5f1ad137d713fca408a98470ea77bddf8916c5f3
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index c9a0b3a..4281fc2 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -8,7 +8,7 @@
settings in the .json files.
- Tests are selected to produce a good coverage.
- Tests are run on the reference model to produce the correct output files.
-- Tests are converted into JSON format and saved to desired output directory.
+- Tests are converted to JSON and/or copied and saved to desired output directory.
"""
import argparse
import copy
@@ -26,6 +26,8 @@
import conformance.model_files as cmf
from conformance.test_select import Operator
from convert2conformance.convert2conformance import main as c2c_main
+from convert2conformance.convert2conformance import OUTPUT_TYPE_DEFAULT
+from convert2conformance.convert2conformance import OUTPUT_TYPES
from distutils.dir_util import copy_tree
logging.basicConfig()
@@ -51,6 +53,10 @@
# standard group will have negative tests generated for it
STANDARD_GENERATOR_GROUP = "standard"
+TEST_VERSION_LATEST = "latest"
+TEST_VERSION_V0_60_0 = "v0.60.0"
+TEST_VERSIONS = (TEST_VERSION_LATEST, TEST_VERSION_V0_60_0)
+
class GenConformanceError(Exception):
"""Generation error reporting exception."""
@@ -214,12 +220,14 @@
return
num_cores = args.num_cores
- run_tests_cmd = "tosa_verif_run_tests"
- ref_cmd_base = ref_cmd = [
- run_tests_cmd,
+ # Use the test runner
+ ref_cmd_base = [
+ "tosa_verif_run_tests",
"--ref-model-path",
str(args.ref_model_path.absolute()),
+ "--schema-path",
+ str(args.schema_path.absolute()),
"-j",
str(num_cores),
"-v",
@@ -243,7 +251,7 @@
)
continue
ref_cmd = ref_cmd_base.copy()
- ref_cmd.append(str(test))
+ ref_cmd.append(str(test.absolute()))
ref_cmds.append(ref_cmd)
fail_string = "UNEXPECTED_FAILURE"
@@ -280,13 +288,14 @@
trim_op_subdir=False,
tags=None,
):
- """Convert tests to JSON and save to output directory."""
+ """Convert/copy tests to output directory."""
if group:
output_dir = output_dir / group
c2c_args_base = ["--strict"]
c2c_args_base.extend(["--schema-path", str(args.schema_path)])
c2c_args_base.extend(["--flatc-path", str(args.flatc_path)])
+ c2c_args_base.extend(["--output-type", args.output_type])
# This op maybe in more than one profile - e.g. tosa_bi and tosa_mi
# even if we are only producing tests for tosa_mi
for op_profile in op_profiles_list:
@@ -349,7 +358,7 @@
logger.error(f"Stopping due to {failed_counter} test conversion errors")
raise (GenConformanceError())
- logger.info("Converted tests to JSON and saved to output directory")
+ logger.info("Converted/copied tests and saved to output directory")
return output_dir
@@ -535,6 +544,20 @@
),
)
parser.add_argument(
+ "--test-version",
+ dest="test_version",
+ choices=TEST_VERSIONS,
+ default=TEST_VERSION_LATEST,
+ help=f"Version of the tests to produce (default is {TEST_VERSION_LATEST})",
+ )
+ parser.add_argument(
+ "--output-type",
+ dest="output_type",
+ choices=OUTPUT_TYPES,
+ default=OUTPUT_TYPE_DEFAULT,
+ help=f"Output file type produced (default is {OUTPUT_TYPE_DEFAULT})",
+ )
+ parser.add_argument(
"--seed",
dest="random_seed",
default=DEFAULT_SEED,
@@ -778,6 +801,10 @@
)
continue
+ if args.test_version == TEST_VERSION_V0_60_0 and op in ("dim",):
+ logger.warning(f"{op} is not in {args.test_version} - skipping")
+ continue
+
op_profiles_list = test_params[op]["profile"]
if (
args.profile != PROFILES_ALL
diff --git a/verif/runner/tosa_test_runner.py b/verif/runner/tosa_test_runner.py
index 30a7168..b348f50 100644
--- a/verif/runner/tosa_test_runner.py
+++ b/verif/runner/tosa_test_runner.py
@@ -4,7 +4,6 @@
import json
from enum import IntEnum
-import conformance.model_files as cmf
import schemavalidation.schemavalidation as sch
from checker.color_print import LogColors
from checker.color_print import print_color
@@ -71,9 +70,7 @@
self.testDir = str(testDirPath)
self.testDirPath = testDirPath
self.testName = self.testDirPath.name
- self.verify_lib_path = cmf.find_tosa_file(
- cmf.TosaFileType.VERIFY_LIBRARY, args.ref_model_path
- )
+ self.verify_lib_path = args.verify_lib_path
set_print_in_color(not args.no_color)
# Stop the result checker printing anything - we will do it
diff --git a/verif/runner/tosa_verif_run_tests.py b/verif/runner/tosa_verif_run_tests.py
index d1755e6..54cb7b2 100644
--- a/verif/runner/tosa_verif_run_tests.py
+++ b/verif/runner/tosa_verif_run_tests.py
@@ -52,6 +52,15 @@
help="Path to TOSA reference model executable",
)
parser.add_argument(
+ "--verify-lib-path",
+ dest="verify_lib_path",
+ type=Path,
+ help=(
+ "Path to TOSA verify library. Defaults to "
+ "the library in the directory of `ref-model-path`"
+ ),
+ )
+ parser.add_argument(
"--operator-fbs",
"--schema-path",
dest="schema_path",
@@ -365,6 +374,10 @@
args.ref_model_path = cmf.find_tosa_file(
cmf.TosaFileType.REF_MODEL, Path("reference_model"), False
)
+ if args.verify_lib_path is None:
+ args.verify_lib_path = cmf.find_tosa_file(
+ cmf.TosaFileType.VERIFY_LIBRARY, args.ref_model_path
+ )
if args.flatc_path is None:
args.flatc_path = cmf.find_tosa_file(
cmf.TosaFileType.FLATC, args.ref_model_path