Initial set up of Main Inference conformance test gen
tosa-verif-build-tests
- option for setting FP values range
- option for recursively finding tests
- change from os.path to Path
tosa_verif_result_check
- option to supply FP tolerance
- output difference and max tolerance on contents mismatch
- change from os.path to Path
MI conformance - contains examples of AVG_POOL2D and CONV2D tests
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I8e1645cd8f10308604400ea53eef723ca163eed7
diff --git a/verif/checker/tosa_result_checker.py b/verif/checker/tosa_result_checker.py
index b7a76b6..1169a95 100644
--- a/verif/checker/tosa_result_checker.py
+++ b/verif/checker/tosa_result_checker.py
@@ -2,7 +2,6 @@
# Copyright (c) 2020-2022, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import argparse
-import os
from enum import Enum
from enum import IntEnum
from enum import unique
@@ -62,37 +61,41 @@
]
##################################
+DEFAULT_FP_TOLERANCE = 1e-3
+
def test_check(
- reference,
- result,
+ reference_path,
+ result_path,
test_name="test",
quantize_tolerance=0,
- float_tolerance=1e-3,
+ float_tolerance=DEFAULT_FP_TOLERANCE,
misc_checks=[],
):
"""Check if the result is the same as the expected reference."""
- if not os.path.isfile(reference):
+ if not reference_path.is_file():
print_color(LogColors.RED, "Reference MISSING FILE {}".format(test_name))
- msg = "Missing reference file: {}".format(reference)
+ msg = "Missing reference file: {}".format(reference_path)
return (TestResult.MISSING_FILE, 0.0, msg)
- if not os.path.isfile(result):
+ if not result_path.is_file():
print_color(LogColors.RED, "Results MISSING FILE {}".format(test_name))
- msg = "Missing result file: {}".format(result)
+ msg = "Missing result file: {}".format(result_path)
return (TestResult.MISSING_FILE, 0.0, msg)
try:
- test_result = np.load(result)
+ test_result = np.load(result_path)
except Exception as e:
print_color(LogColors.RED, "Results INCORRECT FORMAT {}".format(test_name))
- msg = "Incorrect numpy format of {}\nnumpy.load exception: {}".format(result, e)
+ msg = "Incorrect numpy format of {}\nnumpy.load exception: {}".format(
+ result_path, e
+ )
return (TestResult.INCORRECT_FORMAT, 0.0, msg)
try:
- reference_result = np.load(reference)
+ reference_result = np.load(reference_path)
except Exception as e:
print_color(LogColors.RED, "Reference INCORRECT FORMAT {}".format(test_name))
msg = "Incorrect numpy format of {}\nnumpy.load exception: {}".format(
- reference, e
+ reference_path, e
)
return (TestResult.INCORRECT_FORMAT, 0.0, msg)
@@ -109,6 +112,7 @@
# >= 0, allow that special case
test_result = np.squeeze(test_result)
reference_result = np.squeeze(reference_result)
+ difference = None
if np.shape(test_result) != np.shape(reference_result):
print_color(LogColors.RED, "Results MISCOMPARE {}".format(test_name))
@@ -155,6 +159,7 @@
)
)
# Fall-through to below to add failure values
+ difference = reference_result - test_result
elif reference_result.dtype == bool:
assert test_result.dtype == bool
@@ -165,6 +170,7 @@
return (TestResult.PASS, 0.0, "")
msg = "Boolean result does not match"
tolerance = 0.0
+ difference = None
# Fall-through to below to add failure values
# TODO: update for fp16 tolerance
@@ -174,6 +180,7 @@
print_color(LogColors.GREEN, "Results PASS {}".format(test_name))
return (TestResult.PASS, tolerance, "")
msg = "Float result does not match within tolerance of {}".format(tolerance)
+ difference = reference_result - test_result
# Fall-through to below to add failure values
else:
print_color(LogColors.RED, "Results UNSUPPORTED TYPE {}".format(test_name))
@@ -182,16 +189,24 @@
# Fall-through for mismatch failure to add values to msg
print_color(LogColors.RED, "Results MISCOMPARE {}".format(test_name))
- np.set_printoptions(threshold=128)
- msg = "{}\ntest_result: {}\n{}".format(msg, test_result.shape, test_result)
- msg = "{}\nreference_result: {}\n{}".format(
+ np.set_printoptions(threshold=128, edgeitems=2)
+
+ if difference is not None:
+ tolerance_needed = np.amax(np.absolute(difference))
+ msg = "{}\n-- tolerance_needed: {}".format(msg, tolerance_needed)
+
+ msg = "{}\n>> reference_result: {}\n{}".format(
msg, reference_result.shape, reference_result
)
+ msg = "{}\n<< test_result: {}\n{}".format(msg, test_result.shape, test_result)
+
+ if difference is not None:
+ msg = "{}\n!! difference_result: \n{}".format(msg, difference)
return (TestResult.MISMATCH, tolerance, msg)
def main(argv=None):
- """Check that the supplied reference and result files are the same."""
+ """Check that the supplied reference and result files have the same contents."""
parser = argparse.ArgumentParser()
parser.add_argument(
"reference_path", type=Path, help="the path to the reference file to test"
@@ -199,11 +214,14 @@
parser.add_argument(
"result_path", type=Path, help="the path to the result file to test"
)
+ parser.add_argument(
+ "--fp-tolerance", type=float, default=DEFAULT_FP_TOLERANCE, help="FP tolerance"
+ )
args = parser.parse_args(argv)
- ref_path = args.reference_path
- res_path = args.result_path
- result, tolerance, msg = test_check(ref_path, res_path)
+ result, tolerance, msg = test_check(
+ args.reference_path, args.result_path, float_tolerance=args.fp_tolerance
+ )
if result != TestResult.PASS:
print(msg)
diff --git a/verif/conformance/test_select.py b/verif/conformance/test_select.py
index 9e73b0d..c04b7ef 100644
--- a/verif/conformance/test_select.py
+++ b/verif/conformance/test_select.py
@@ -135,10 +135,39 @@
test_dir: the directory where the tests for all operators can be found
config: a dictionary with:
- "params" - mappings of parameter names to the values to select
+ "params" - a dictionary with mappings of parameter names to the values
+ to select (a sub-set of expected values for instance)
"permutes" - a list of parameter names to be permuted
+ "preselected" - a list of dictionaries containing parameter names and
+ pre-chosen values
+ "sparsity" - a dictionary of parameter names with a sparsity value
"errorifs" - list of ERRORIF case names to be selected (negative test)
negative: bool indicating if negative testing is being selected (ERRORIF tests)
+
+ EXAMPLE CONFIG:
+ "params": {
+ "output_type": [
+ "outi8",
+ "outb"
+ ]
+ },
+ "permutes": [
+ "shape",
+ "type"
+ ],
+ "sparsity": {
+ "pad": 15
+ },
+ "preselected": [
+ {
+ "shape": "6",
+ "type": "i8",
+ "pad": "pad00"
+ }
+ ],
+ "errorifs": [
+ "InputZeroPointNotZero"
+ ]
"""
assert isinstance(
self.name, str
diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json
new file mode 100644
index 0000000..656cdde
--- /dev/null
+++ b/verif/conformance/tosa_main_profile_ops_info.json
@@ -0,0 +1,96 @@
+{
+ "avg_pool2d": {
+ "group": "tensor",
+ "generator_args": [
+ [
+ "--target-dtype",
+ "fp32",
+ "--target-dtype",
+ "fp16",
+ "--target-dtype",
+ "bf16",
+ "--fp-values-range",
+ "-2.0,2.0",
+ "--target-shape",
+ "1,49,42,28",
+ "--target-shape",
+ "3,11,44,3",
+ "--allow-pooling-and-conv-oversizes"
+ ],
+ [
+ "--target-dtype",
+ "fp32",
+ "--fp-values-range",
+ "-2.0,2.0",
+ "--target-shape",
+ "1,65535,5,1",
+ "--target-shape",
+ "1,3,65537,1",
+ "--allow-pooling-and-conv-oversizes"
+ ]
+ ],
+ "params": {
+ },
+ "permutes": [
+ "shape",
+ "type",
+ "accum_type"
+ ],
+ "sparsity": {
+ "pad": 17
+ },
+ "profile": [
+ "tosa-mi"
+ ]
+ },
+ "conv2d": {
+ "group": "tensor",
+ "generator_args": [
+ [
+ "--target-dtype",
+ "fp32",
+ "--target-dtype",
+ "fp16",
+ "--target-dtype",
+ "bf16",
+ "--fp-values-range",
+ "-2.0,2.0",
+ "--target-shape",
+ "1,49,42,28",
+ "--target-shape",
+ "1,11,44,13",
+ "--allow-pooling-and-conv-oversizes"
+ ],
+ [
+ "--target-dtype",
+ "fp32",
+ "--fp-values-range",
+ "-2.0,2.0",
+ "--target-shape",
+ "1,65535,4,1",
+ "--target-shape",
+ "1,5,65536,1",
+ "--max-conv-dilation",
+ "1",
+ "--allow-pooling-and-conv-oversizes"
+ ]
+ ],
+ "params": {
+ "shape": [],
+ "type": [],
+ "kernel": [],
+ "stride": [],
+ "pad": [],
+ "dilation": []
+ },
+ "permutes": [
+ "kernel",
+ "shape",
+ "type",
+ "pad"
+ ],
+ "profile": [
+ "tosa-mi"
+ ]
+ }
+}
\ No newline at end of file
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index 7032ad4..8f23f57 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -35,8 +35,15 @@
"operator_test_params": "tosa_base_profile_ops_info.json",
"framework_tests": "tosa_base_profile_framework_ops_info.json",
"exclude_types": [],
- }
+ },
+ "tosa-mi": {
+ # Note: This is just the extra tests not in the base profile!
+ "operator_test_params": "tosa_main_profile_ops_info.json",
+ "framework_tests": "tosa_main_profile_framework_ops_info.json",
+ "exclude_types": [],
+ },
}
+PROFILES_ALL = "all"
LOCATION_REF_MODEL_BINARY = Path("build/reference_model/tosa_reference_model")
@@ -181,7 +188,7 @@
return tests
-def generate_results(args, operator, op_build_dir, tests=None):
+def generate_results(args, profile, operator, op_build_dir, tests=None):
"""Run tests on reference model and save result to the test directory."""
num_cores = args.num_cores
run_tests_cmd = "tosa_verif_run_tests"
@@ -201,7 +208,7 @@
if not tests:
# Do not need to run ERRORIF tests as they don't have result files
tests = _get_all_tests_list(
- args.profile, op_build_dir, operator, exclude_negative_tests=True
+ profile, op_build_dir, operator, exclude_negative_tests=True
)
for test in tests:
@@ -232,10 +239,11 @@
def convert_tests(
args,
+ profile,
operator,
op_build_dir,
output_dir,
- profiles,
+ op_profiles_list,
tests=None,
group=None,
trim_op_subdir=False,
@@ -247,8 +255,10 @@
output_dir = output_dir / group
ref_cmd_base = ["--ref-model-directory", str(ref_model_dir)]
- for profile in profiles:
- ref_cmd_base.extend(["--profile", profile])
+ # This op maybe in more than one profile - e.g. tosa_bi and tosa_mi
+ # even if we are only producing tests for tosa_mi
+ for op_profile in op_profiles_list:
+ ref_cmd_base.extend(["--profile", op_profile])
if args.framework_schema:
ref_cmd_base.extend(["--framework-schema", str(args.framework_schema)])
ref_cmd_base.append("--output-directory")
@@ -256,8 +266,8 @@
ref_cmds = []
if not tests:
- tests = _get_all_tests_list(args.profile, op_build_dir, operator)
- logger.info(f"Converting all {args.profile} profile tests")
+ tests = _get_all_tests_list(profile, op_build_dir, operator)
+ logger.info(f"Converting all {profile} profile tests")
# Controls if we copy the tests in their operator sub-directory or not
output_dir_relative_pos = -1 if trim_op_subdir else -2
@@ -303,16 +313,19 @@
return output_dir
-def get_op_tests_selection(args, operator, op_build_dir, test_params, negative=False):
+def get_op_tests_selection(
+ args, profile, operator, op_build_dir, test_params, negative=False
+):
"""Use test picker to get subsection of tests generated."""
assert operator in test_params
+ logger.info("Choosing {} tests".format(("negative" if negative else "positive")))
try:
op_params = test_params[operator]
op = Operator.registry[operator](
op_build_dir,
op_params,
negative,
- exclude_types=PROFILE_OPS_INFO[args.profile]["exclude_types"],
+ exclude_types=PROFILE_OPS_INFO[profile]["exclude_types"],
)
except KeyError:
logger.error(f"{operator} operator is not supported by test_select")
@@ -321,11 +334,11 @@
return op.select_tests()
-def check_op_tests(args, operator, output_dir):
+def check_op_tests(args, profile, operator, output_dir):
"""Move test folders than contain files larger than 30MB to new directory."""
destination_dir = str(args.output_dir) + "_large_files"
- tests = _get_all_tests_list(args.profile, output_dir, operator, include_all=True)
+ tests = _get_all_tests_list(profile, output_dir, operator, include_all=True)
if not tests:
logger.error(
f"Couldn't find any tests to size check for {operator} in {output_dir}"
@@ -408,6 +421,7 @@
"""Parse the arguments."""
parser = argparse.ArgumentParser()
profiles = list(PROFILE_OPS_INFO.keys())
+ profiles.append(PROFILES_ALL)
parser.add_argument(
"--profile",
dest="profile",
@@ -555,7 +569,6 @@
logging.getLogger("test_select").setLevel(loglevel)
logging.getLogger("convert2conformance").setLevel(loglevel)
- print(f"Creating conformance tests for TOSA {args.profile} profile")
print(f"Output directory: {args.output_dir}")
if args.random_seed != DEFAULT_SEED:
@@ -567,147 +580,172 @@
logger.debug(f"Creating build directory: {args.build_dir}")
args.build_dir.mkdir(parents=True, exist_ok=True)
+ # TODO: For tosa-mi should really generate tosa-bi profile as well
+ # - for now leave it as subset instead of as superset (for testing)
+ if args.profile == PROFILES_ALL:
+ profiles = list(PROFILE_OPS_INFO.keys())
+ else:
+ profiles = [args.profile]
+
try:
- # Framework unit tests
- if args.unit_tests in ["framework", "both"]:
- logger.debug("Creating FRAMEWORK unit tests")
- test_picks_file = (
- args.param_json_dir / PROFILE_OPS_INFO[args.profile]["framework_tests"]
- )
- try:
- with open(test_picks_file, "r") as fd:
- test_picks = json.load(fd)
- except Exception as e:
- logger.error(
- f"Couldn't load framework tests info - {test_picks_file}: {e}"
+ for profile in profiles:
+ print(f"Creating conformance tests for TOSA {profile} profile")
+ # Framework unit tests
+ if args.unit_tests in ["framework", "both"]:
+ logger.debug("Creating FRAMEWORK unit tests")
+ test_picks_file = (
+ args.param_json_dir / PROFILE_OPS_INFO[profile]["framework_tests"]
)
- return 1
-
- operators = args.operators
- if not operators:
- # Create tests for all the operators
- operators = list(test_picks.keys())
-
- root_output_dir = args.output_dir / "frameworks" / "tflite" / "operators"
- for op in operators:
- if op not in test_picks:
- logger.warning(
- f"Framework op {op} not found in {test_picks_file} - skipping"
+ try:
+ with open(test_picks_file, "r") as fd:
+ test_picks = json.load(fd)
+ except Exception as e:
+ logger.error(
+ f"Couldn't load framework tests info - {test_picks_file}: {e}"
)
- continue
+ return 1
- logger.debug(f"Copying and renaming {op}")
- framework_test_dir = copy_rename_framework_tests(args, op, test_picks)
- profiles = test_picks[op]["profile"]
- if args.convert_all_tests:
- logger.debug("Running and converting all framework tests")
+ operators = args.operators
+ if not operators:
+ # Create tests for all the operators
+ operators = list(test_picks.keys())
+
+ root_output_dir = (
+ args.output_dir / "frameworks" / "tflite" / "operators"
+ )
+ for op in operators:
+ logger.info(f"FRAMEWORK OP: {op}")
+ if op not in test_picks:
+ logger.warning(
+ f"Framework op {op} not found in {test_picks_file} - skipping"
+ )
+ continue
+
+ op_profiles_list = test_picks[op]["profile"]
+ if (
+ args.profile != PROFILES_ALL
+ and args.profile not in op_profiles_list
+ ):
+ # Skip this operator as not part of the profile chosen
+ logger.debug(f"Skipping {op} as not part of {args.profile}")
+ continue
+
+ logger.debug(f"Copying and renaming {op}")
+ framework_test_dir = copy_rename_framework_tests(
+ args, op, test_picks
+ )
+
+ if args.convert_all_tests:
+ logger.debug("Running and converting all framework tests")
+ framework_tests = None # Don't select any
+ else:
+ logger.debug("Running and converting selected framework tests")
+ framework_tests = get_framework_tests_selection(
+ args, op, test_picks, framework_test_dir
+ )
convert_tests(
args,
+ profile,
op,
framework_test_dir,
root_output_dir,
- profiles,
- trim_op_subdir=True,
- )
- else:
- framework_tests = get_framework_tests_selection(
- args, op, test_picks, framework_test_dir
- )
- convert_tests(
- args,
- op,
- framework_test_dir,
- root_output_dir,
- profiles,
+ op_profiles_list,
tests=framework_tests,
trim_op_subdir=True,
)
- # Operator unit tests
- if args.unit_tests in ["operator", "both"]:
- logger.debug("Creating OPERATOR unit tests")
- test_params_file = (
- args.param_json_dir
- / PROFILE_OPS_INFO[args.profile]["operator_test_params"]
- )
- try:
- with open(test_params_file, "r") as fd:
- test_params = json.load(fd)
- except Exception as e:
- logger.error(
- f"Couldn't load operator test params - {test_params_file}: {e}"
+ # Operator unit tests
+ if args.unit_tests in ["operator", "both"]:
+ logger.debug("Creating OPERATOR unit tests")
+ test_params_file = (
+ args.param_json_dir
+ / PROFILE_OPS_INFO[profile]["operator_test_params"]
)
- return 1
-
- operators = args.operators
- if not operators:
- # Create tests for all the operators
- operators = list(test_params.keys())
-
- for op in operators:
- if op not in test_params:
- logger.warning(
- f"{op} operator parameters not found in {test_params_file} - skipping"
+ try:
+ with open(test_params_file, "r") as fd:
+ test_params = json.load(fd)
+ except Exception as e:
+ logger.error(
+ f"Couldn't load operator test params - {test_params_file}: {e}"
)
- continue
+ return 1
- if (
- args.test_type == "negative"
- and "no_negative_tests" in test_params[op]
- and test_params[op]["no_negative_tests"]
- ):
- logger.warning(f"No negative tests for {op}")
- continue
+ operators = args.operators
+ if not operators:
+ # Create tests for all the operators
+ operators = list(test_params.keys())
- op_build_dir = build_op_tests(args, op, test_params)
+ for op in operators:
+ logger.info(f"OPERATOR: {op}")
+ if op not in test_params:
+ logger.warning(
+ f"{op} operator parameters not found in {test_params_file} - skipping"
+ )
+ continue
- operator_group = test_params[op]["group"]
- root_output_dir = args.output_dir / "operators"
- profiles = test_params[op]["profile"]
- if args.convert_all_tests:
- logger.debug(f"Running and converting all {op} tests")
- generate_results(args, op, op_build_dir)
+ if (
+ args.test_type == "negative"
+ and "no_negative_tests" in test_params[op]
+ and test_params[op]["no_negative_tests"]
+ ):
+ logger.warning(f"No negative tests for {op}")
+ continue
+
+ op_profiles_list = test_params[op]["profile"]
+ if (
+ args.profile != PROFILES_ALL
+ and args.profile not in op_profiles_list
+ ):
+ # Skip this operator as not part of the profile chosen
+ logger.debug(f"Skipping {op} as not part of {args.profile}")
+ continue
+
+ op_build_dir = build_op_tests(args, op, test_params)
+
+ operator_group = test_params[op]["group"]
+ root_output_dir = args.output_dir / "operators"
+ if args.convert_all_tests:
+ logger.debug(f"Running and converting all {op} tests")
+ generate_results(args, profile, op, op_build_dir)
+ operator_test_list = None
+ else:
+ logger.debug(f"Running and converting selection of {op} tests")
+ if args.test_type in ["positive", "both"]:
+ tests_gen, tests_gen2 = tee(
+ get_op_tests_selection(
+ args, profile, op, op_build_dir, test_params
+ )
+ )
+ generate_results(args, profile, op, op_build_dir, tests_gen)
+ operator_test_list = list(tests_gen2)
+ else:
+ operator_test_list = []
+ if args.test_type in ["negative", "both"] and (
+ "no_negative_tests" not in test_params[op]
+ or not test_params[op]["no_negative_tests"]
+ ):
+ operator_test_list.extend(
+ get_op_tests_selection(
+ args,
+ profile,
+ op,
+ op_build_dir,
+ test_params,
+ negative=True,
+ )
+ )
output_dir = convert_tests(
args,
+ profile,
op,
op_build_dir,
root_output_dir,
- profiles,
+ op_profiles_list,
+ tests=operator_test_list,
group=operator_group,
)
- else:
- if args.test_type in ["positive", "both"]:
- tests_gen1, tests_gen2 = tee(
- get_op_tests_selection(args, op, op_build_dir, test_params)
- )
- generate_results(args, op, op_build_dir, tests_gen1)
- output_dir = convert_tests(
- args,
- op,
- op_build_dir,
- root_output_dir,
- profiles,
- tests=tests_gen2,
- group=operator_group,
- )
- if args.test_type in ["negative", "both"] and (
- "no_negative_tests" not in test_params[op]
- or not test_params[op]["no_negative_tests"]
- ):
- negative_tests = get_op_tests_selection(
- args, op, op_build_dir, test_params, negative=True
- )
- output_dir = convert_tests(
- args,
- op,
- op_build_dir,
- root_output_dir,
- profiles,
- tests=negative_tests,
- group=operator_group,
- )
- if not args.keep_large_files:
- check_op_tests(args, op, output_dir)
+ if not args.keep_large_files:
+ check_op_tests(args, profile, op, output_dir)
except GenConformanceError:
return 1
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index 95e06ed..f3ca512 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -36,6 +36,9 @@
self.quantGen = TosaQuantGen()
# Force makeShape to do a specific starting shape
self.targetted_shape = None
+ # Work out floating point range
+ self.random_fp_low = min(args.tensor_fp_value_range)
+ self.random_fp_high = max(args.tensor_fp_value_range)
def createSerializer(self, opName, testPath):
self.testPath = os.path.join(opName, testPath)
@@ -84,13 +87,25 @@
self.rng.integers(low=-(1 << 47), high=(1 << 47), size=shape)
)
elif dtype == DType.FP16:
- return np.float16(self.rng.random(size=shape))
+ return np.float16(
+ self.rng.uniform(
+ low=self.random_fp_low, high=self.random_fp_high, size=shape
+ )
+ )
elif dtype == DType.BF16:
- f32_tensor = np.float32(self.rng.random(size=shape))
+ f32_tensor = np.float32(
+ self.rng.uniform(
+ low=self.random_fp_low, high=self.random_fp_high, size=shape
+ )
+ )
# Floor the last 16 bits of each f32 value
return np.float32(vect_f32_to_bf16(f32_tensor))
elif dtype == DType.FP32:
- return np.float32(self.rng.random(size=shape))
+ return np.float32(
+ self.rng.uniform(
+ low=self.random_fp_low, high=self.random_fp_high, size=shape
+ )
+ )
else:
raise Exception("Unrecognized Dtype: {}".format(dtype))
@@ -135,12 +150,17 @@
def getRandNumberDType(self, dtype):
if dtype == DType.FP32:
- return self.rng.random()
+ return np.float32(
+ self.rng.uniform(low=self.random_fp_low, high=self.random_fp_high)
+ )
elif dtype == DType.FP16:
- rand_f32 = self.rng.random()
- return np.float16(rand_f32)
+ return np.float16(
+ self.rng.uniform(low=self.random_fp_low, high=self.random_fp_high)
+ )
elif dtype == DType.BF16:
- rand_f32 = self.rng.random()
+ rand_f32 = np.float32(
+ self.rng.uniform(low=self.random_fp_low, high=self.random_fp_high)
+ )
return vect_f32_to_bf16(rand_f32)
elif dtype == DType.BOOL:
return self.rng.choice([False, True])
diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py
index ab78b1a..bc1ec8e 100644
--- a/verif/generator/tosa_verif_build_tests.py
+++ b/verif/generator/tosa_verif_build_tests.py
@@ -2,20 +2,24 @@
# SPDX-License-Identifier: Apache-2.0
import argparse
import re
+import sys
from generator.tosa_test_gen import TosaTestGen
from serializer.tosa_serializer import dtype_str_to_val
from serializer.tosa_serializer import DTypeNames
+OPTION_FP_VALUES_RANGE = "--fp-values-range"
+
# Used for parsing a comma-separated list of integers in a string
# to an actual list of integers
-def str_to_list(in_s):
+def str_to_list(in_s, is_float=False):
"""Converts a comma-separated list of string integers to a python list of ints"""
lst = in_s.split(",")
out_list = []
for i in lst:
- out_list.append(int(i))
+ val = float(i) if is_float else int(i)
+ out_list.append(val)
return out_list
@@ -25,6 +29,26 @@
def parseArgs(argv):
+ """Parse the command line arguments."""
+ if argv is None:
+ argv = sys.argv[1:]
+
+ if OPTION_FP_VALUES_RANGE in argv:
+ # Argparse fix for hyphen (minus values) in argument values
+ # convert "ARG VAL" into "ARG=VAL"
+ # Example --fp-values-range -2.0,2.0 -> --fp-values-range=-2.0,2.0
+ new_argv = []
+ idx = 0
+ while idx < len(argv):
+ arg = argv[idx]
+ if arg == OPTION_FP_VALUES_RANGE and idx + 1 < len(argv):
+ val = argv[idx + 1]
+ if val.startswith("-"):
+ arg = f"{arg}={val}"
+ idx += 1
+ new_argv.append(arg)
+ idx += 1
+ argv = new_argv
parser = argparse.ArgumentParser()
parser.add_argument(
@@ -61,6 +85,14 @@
)
parser.add_argument(
+ OPTION_FP_VALUES_RANGE,
+ dest="tensor_fp_value_range",
+ default="0.0,1.0",
+ type=lambda x: str_to_list(x, is_float=True),
+ help="Min,Max range of floating point tensor values",
+ )
+
+ parser.add_argument(
"--max-batch-size",
dest="max_batch_size",
default=1,
@@ -132,7 +164,7 @@
help="Upper limit on width and height output dimensions for `resize` op. Default: 1000",
)
- # Targetting a specific shape/rank/dtype
+ # Targeting a specific shape/rank/dtype
parser.add_argument(
"--target-shape",
dest="target_shapes",
diff --git a/verif/runner/tosa_refmodel_sut_run.py b/verif/runner/tosa_refmodel_sut_run.py
index 6acaaf4..95f6e7b 100644
--- a/verif/runner/tosa_refmodel_sut_run.py
+++ b/verif/runner/tosa_refmodel_sut_run.py
@@ -21,9 +21,9 @@
class TosaSUTRunner(TosaTestRunner):
"""TOSA Reference Model runner."""
- def __init__(self, args, runnerArgs, testDir):
+ def __init__(self, args, runnerArgs, testDirPath):
"""Initialize using the given test details."""
- super().__init__(args, runnerArgs, testDir)
+ super().__init__(args, runnerArgs, testDirPath)
def runTestGraph(self):
"""Run the test on the reference model."""
diff --git a/verif/runner/tosa_test_runner.py b/verif/runner/tosa_test_runner.py
index 65931d8..d8c2a87 100644
--- a/verif/runner/tosa_test_runner.py
+++ b/verif/runner/tosa_test_runner.py
@@ -33,47 +33,49 @@
class TosaTestRunner:
"""TOSA Test Runner template class for systems under test."""
- def __init__(self, args, runnerArgs, testDir):
+ def __init__(self, args, runnerArgs, testDirPath):
"""Initialize and load JSON meta data file."""
self.args = args
self.runnerArgs = runnerArgs
- self.testDir = testDir
- self.testName = Path(self.testDir).name
+ self.testDir = str(testDirPath)
+ self.testDirPath = testDirPath
+ self.testName = self.testDirPath.name
set_print_in_color(not args.no_color)
# Check if we want to run binary and if its already converted
- descFilePath = Path(testDir, "desc.json")
- descBinFilePath = Path(testDir, "desc_binary.json")
+ descFilePath = testDirPath / "desc.json"
+ descBinFilePath = testDirPath / "desc_binary.json"
if args.binary:
if descBinFilePath.is_file():
descFilePath = descBinFilePath
try:
# Load the json test file
- with open(descFilePath, "r") as fd:
+ with descFilePath.open("r") as fd:
self.testDesc = json.load(fd)
except Exception as e:
raise TosaTestInvalid(str(descFilePath), e)
# Convert to binary if needed
- tosaFilePath = Path(testDir, self.testDesc["tosa_file"])
+ tosaFilePath = testDirPath / self.testDesc["tosa_file"]
if args.binary and tosaFilePath.suffix == ".json":
# Convert tosa JSON to binary
json2fbbin.json_to_fbbin(
Path(args.flatc_path),
Path(args.operator_fbs),
tosaFilePath,
- Path(testDir),
+ testDirPath,
)
# Write new desc_binary file
self.testDesc["tosa_file"] = tosaFilePath.stem + ".tosa"
- with open(descBinFilePath, "w") as fd:
+ with descBinFilePath.open("w") as fd:
json.dump(self.testDesc, fd, indent=2)
descFilePath = descBinFilePath
# Set location of desc.json (or desc_binary.json) file in use
self.descFile = str(descFilePath)
+ self.descFilePath = descFilePath
def skipTest(self):
"""Check if the test is skipped due to test type or profile selection."""
@@ -109,9 +111,9 @@
for resultNum, resultFileName in enumerate(self.testDesc["ofm_file"]):
if "expected_result_file" in self.testDesc:
try:
- conformanceFile = Path(
- self.testDir,
- self.testDesc["expected_result_file"][resultNum],
+ conformanceFilePath = (
+ self.testDirPath
+ / self.testDesc["expected_result_file"][resultNum]
)
except IndexError:
result = TosaTestRunner.Result.INTERNAL_ERROR
@@ -122,14 +124,14 @@
print(msg)
break
else:
- conformanceFile = None
- resultFile = Path(self.testDir, resultFileName)
+ conformanceFilePath = None
+ resultFilePath = self.testDirPath / resultFileName
- if conformanceFile:
+ if conformanceFilePath:
print_result_line = False # Checker will print one for us
chkResult, tolerance, msg = test_check(
- str(conformanceFile),
- str(resultFile),
+ conformanceFilePath,
+ resultFilePath,
test_name=self.testName,
)
# Change EXPECTED_PASS assumption if we have any failures
@@ -140,18 +142,18 @@
print(msg)
else:
# No conformance file to verify, just check results file exists
- if not resultFile.is_file():
+ if not resultFilePath.is_file():
result = TosaTestRunner.Result.UNEXPECTED_FAILURE
- msg = "Results file is missing: {}".format(resultFile)
+ msg = "Results file is missing: {}".format(resultFilePath)
messages.append(msg)
print(msg)
- if resultFile.is_file():
- # Move the resultFile to allow subsequent system under
+ if resultFilePath.is_file():
+ # Move the resultFilePath to allow subsequent system under
# tests to create them and to test they have been created
- resultFile = resultFile.rename(
- resultFile.with_suffix(
- ".{}{}".format(self.__module__, resultFile.suffix)
+ resultFilePath = resultFilePath.rename(
+ resultFilePath.with_suffix(
+ ".{}{}".format(self.__module__, resultFilePath.suffix)
)
)
diff --git a/verif/runner/tosa_verif_run_tests.py b/verif/runner/tosa_verif_run_tests.py
index 77394cc..ddb32a4 100644
--- a/verif/runner/tosa_verif_run_tests.py
+++ b/verif/runner/tosa_verif_run_tests.py
@@ -2,7 +2,6 @@
# Copyright (c) 2020-2022, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import argparse
-import glob
import importlib
import os
import queue
@@ -40,6 +39,13 @@
help="File containing list of tests to run (one per line)",
)
parser.add_argument(
+ "-r",
+ "--recursive",
+ dest="recursive_tests",
+ action="store_true",
+ help="Recursively search for tests",
+ )
+ parser.add_argument(
"--operator-fbs",
dest="operator_fbs",
default="conformance_tests/third_party/serialization_lib/schema/tosa.fbs",
@@ -146,27 +152,28 @@
EXCLUSION_PREFIX = ["test", "model", "desc"]
-def convert2Numpy(testDir):
+def convert2Numpy(test_path):
"""Convert all the JSON numpy files back into binary numpy."""
- jsons = glob.glob(os.path.join(testDir, "*.json"))
+ jsons = test_path.glob("*.json")
for json in jsons:
for exclude in EXCLUSION_PREFIX:
- if os.path.basename(json).startswith(exclude):
- json = ""
+ if json.name.startswith(exclude):
+ json = None
+ break
if json:
- # debug print("Converting " + json)
- json2numpy.json_to_npy(Path(json))
+ # debug print(f"Converting {json}")
+ json2numpy.json_to_npy(json)
def workerThread(task_queue, runnerList, args, result_queue):
"""Worker thread that runs the next test from the queue."""
while True:
try:
- test = task_queue.get(block=False)
+ test_path = task_queue.get(block=False)
except queue.Empty:
break
- if test is None:
+ if test_path is None:
break
msg = ""
@@ -176,21 +183,25 @@
start_time = datetime.now()
# Set up system under test runner
runnerName = runnerModule.__name__
- runner = runnerModule.TosaSUTRunner(args, runnerArgs, test)
+ runner = runnerModule.TosaSUTRunner(args, runnerArgs, test_path)
skip, reason = runner.skipTest()
if skip:
msg = "Skipping {} test".format(reason)
- print("{} {}".format(msg, test))
+ print("{} {}".format(msg, test_path))
rc = TosaTestRunner.Result.SKIPPED
else:
# Convert JSON data files into numpy format on first pass
if not converted:
- convert2Numpy(test)
+ convert2Numpy(test_path)
converted = True
if args.verbose:
- print("Running runner {} with test {}".format(runnerName, test))
+ print(
+ "Running runner {} with test {}".format(
+ runnerName, test_path
+ )
+ )
try:
grc, gmsg = runner.runTestGraph()
rc, msg = runner.testResult(grc, gmsg)
@@ -220,7 +231,9 @@
rc = TosaTestRunner.Result.INTERNAL_ERROR
finally:
end_time = datetime.now()
- result_queue.put((runnerName, test, rc, msg, end_time - start_time))
+ result_queue.put(
+ (runnerName, test_path, rc, msg, end_time - start_time)
+ )
task_queue.task_done()
@@ -262,10 +275,10 @@
xunit_suite = xunit_result.create_suite(runner)
# Sort by test name
- for test, rc, msg, time_delta in sorted(
+ for test_path, rc, msg, time_delta in sorted(
resultLists[runner], key=lambda tup: tup[0]
):
- test_name = test
+ test_name = str(test_path)
xt = xunit.xunit_test(test_name, runner)
xt.time = str(
@@ -293,12 +306,27 @@
xunit_result.write_results(xunitFile)
+def getTestsInPath(path):
+ # Recursively find any tests in this directory
+ desc_path = path / "desc.json"
+ if desc_path.is_file():
+ return [path]
+ elif path.is_dir():
+ path_list = []
+ for p in path.glob("*"):
+ path_list.extend(getTestsInPath(p))
+ return path_list
+ else:
+ return []
+
+
def main(argv=None):
"""Start worker threads to do the testing and outputs the results."""
args = parseArgs(argv)
- if TOSA_REFMODEL_RUNNER in args.sut_module and not os.path.isfile(
- args.ref_model_path
+ if (
+ TOSA_REFMODEL_RUNNER in args.sut_module
+ and not Path(args.ref_model_path).is_file()
):
print(
"Argument error: Reference Model not found ({})".format(args.ref_model_path)
@@ -307,7 +335,7 @@
if args.test_list_file:
try:
- with open(args.test_list_file) as f:
+ with args.test_list_file.open("r") as f:
args.test = f.read().splitlines()
except Exception as e:
print(
@@ -323,12 +351,21 @@
taskQueue = queue.Queue()
resultQueue = queue.Queue()
- for t in args.test:
- if os.path.isfile(t):
- if not os.path.basename(t) == "README":
- print("Warning: Skipping test {} as not a valid directory".format(t))
+ for tdir in args.test:
+ tpath = Path(tdir)
+ if tpath.is_file():
+ if tpath.name != "README":
+ print(
+ "Warning: Skipping test {} as not a valid directory".format(tpath)
+ )
else:
- taskQueue.put((t))
+ if args.recursive_tests:
+ tpath_list = getTestsInPath(tpath)
+ else:
+ tpath_list = [tpath]
+
+ for t in tpath_list:
+ taskQueue.put((t))
print(
"Running {} tests on {} system{} under test".format(
@@ -356,7 +393,7 @@
while True:
try:
- runner, test, rc, msg, time_delta = resultQueue.get(block=False)
+ runner, test_path, rc, msg, time_delta = resultQueue.get(block=False)
resultQueue.task_done()
except queue.Empty:
break
@@ -368,7 +405,7 @@
msg = "{} ...\nskipped {} bytes\n... {}".format(
msg[:half], trimmed, msg[-half:]
)
- resultLists[runner].append((test, rc, msg, time_delta))
+ resultLists[runner].append((test_path, rc, msg, time_delta))
results[runner][rc] += 1
createXUnitResults(args.xunit_file, runnerList, resultLists, args.verbose)
diff --git a/verif/tests/test_tosa_refmodel.py b/verif/tests/test_tosa_refmodel.py
index 50ff1ab..1f9cd3e 100644
--- a/verif/tests/test_tosa_refmodel.py
+++ b/verif/tests/test_tosa_refmodel.py
@@ -240,8 +240,8 @@
# Check Numpy result versus refmodel
check_result, tolerance, msg = tosa_check(
- str(result_file),
- str(ofm_file),
+ result_file,
+ ofm_file,
test_name=test_dir.name,
misc_checks=misc_checks,
)
diff --git a/verif/tests/tosa_dummy_sut_run.py b/verif/tests/tosa_dummy_sut_run.py
index fffcfa1..344aa2d 100644
--- a/verif/tests/tosa_dummy_sut_run.py
+++ b/verif/tests/tosa_dummy_sut_run.py
@@ -7,9 +7,9 @@
class TosaSUTRunner(TosaTestRunner):
"""TOSA dummy SUT runner."""
- def __init__(self, args, runnerArgs, testDir):
+ def __init__(self, args, runnerArgs, testDirPath):
"""Initialize using the given test details."""
- super().__init__(args, runnerArgs, testDir)
+ super().__init__(args, runnerArgs, testDirPath)
def runTestGraph(self):
"""Nothing run as this is a dummy SUT that does nothing."""