Add profile to all created conformance tests.
Add supported profiles for each test in convert2conformance and
tosa_verif_conformance_generator.
Enable filtering of profile tests on running in tosa_verif_run_tests.
Reorganize arguments in conformance_generator to have more important
ones first.
Change-Id: Ie6e5b68727adb3c39b04aa482dd6433788f7bcc9
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py
index aa41ff1..06726ba 100755
--- a/scripts/convert2conformance/convert2conformance.py
+++ b/scripts/convert2conformance/convert2conformance.py
@@ -30,6 +30,8 @@
NAME_CONFORMANCE_RESULT_PREFIX = "Conformance-"
NAME_REFMODEL_RUN_RESULT_SUFFIX = ".runner.tosa_refmodel_sut_run.npy"
+PROFILES_LIST = ["tosa-bi", "tosa-mi"]
+
def parse_args(argv):
"""Parse the arguments."""
@@ -69,9 +71,18 @@
help="Framework schema needed to convert framework models",
)
parser.add_argument(
+ "--profile",
+ dest="profile",
+ choices=PROFILES_LIST,
+ action="append",
+ required=True,
+ help="Profiles this test is suitable for. May be repeated",
+ )
+ parser.add_argument(
"-v", "--verbose", dest="verbose", action="store_true", help="Verbose operation"
)
args = parser.parse_args(argv)
+
return args
@@ -138,7 +149,11 @@
def update_desc_json(
- test_dir: Path, test_desc, output_dir: Optional[Path] = None, create_result=True
+ test_dir: Path,
+ test_desc,
+ output_dir: Optional[Path] = None,
+ create_result=True,
+ profiles=None,
):
"""Update the desc.json format for conformance and optionally create result."""
ofm_files = []
@@ -171,6 +186,13 @@
if not test_desc["expected_failure"]:
# Output expected result file for conformance if expected pass
test_desc["expected_result_file"] = cfm_files
+
+ # Add supported profiles
+ if profiles is None:
+ # Assume base profile
+ profiles = [PROFILES_LIST[0]]
+ test_desc["profile"] = profiles
+
return test_desc
@@ -289,7 +311,11 @@
# Update desc.json and convert result files to JSON
test_desc = update_desc_json(
- desc_filename.parent, test_desc, output_dir=args.output_dir, create_result=True
+ desc_filename.parent,
+ test_desc,
+ output_dir=args.output_dir,
+ create_result=True,
+ profiles=args.profile,
)
if not test_desc:
# Error from conversion/update
diff --git a/verif/conformance/tosa_base_profile_framework_ops_info.json b/verif/conformance/tosa_base_profile_framework_ops_info.json
index 7beb2dd..c8bcb00 100644
--- a/verif/conformance/tosa_base_profile_framework_ops_info.json
+++ b/verif/conformance/tosa_base_profile_framework_ops_info.json
@@ -3,6 +3,10 @@
"tests": [
"add_1_qi16",
"add_1x32x32x8_qi8"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"average_pool_2d": {
@@ -15,6 +19,10 @@
"average_pool_2d_1x7x7x9_qi8_st22_padSAME_kern11",
"average_pool_2d_1x32x32x8_qu8_st12_padVALID_kern12",
"average_pool_2d_1x8x4x17_qu8_st21_padVALID_kern21"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"concatenation": {
@@ -31,6 +39,10 @@
"alternate_names": [
"concat",
"concatv2"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"conv_2d": {
@@ -51,6 +63,10 @@
"conv_2d_relu_n1_to_1_5x5_1x7x7x9_qi8",
"conv_2d_tanh_3x3_1x7x7x9_qi8",
"conv_2d_tanh_5x5_1x32x32x8_qu8"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"depthwise_conv_2d": {
@@ -63,6 +79,10 @@
"depthwise_conv_2d_5x5_1x32x32x8_qi8_st11_padSAME_dilat12",
"depthwise_conv_2d_bias_1x1_1x4x4x4_qu8_st11_padSAME_dilat11",
"depthwise_conv_2d_bias_3x3_1x32x32x8_qu8_st11_padVALID_dilat22"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"fully_connected": {
@@ -73,6 +93,10 @@
"fully_connected_1x4x4x4_qi8",
"fully_connected_13x21x3_qu8",
"fully_connected_14x19_qi8"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"leaky_relu": {
@@ -81,6 +105,10 @@
"leaky_relu_14x19_qi8_0",
"leaky_relu_13x21x3_qu8_0",
"leaky_relu_1x4x8x19_qi16_0"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"logistic": {
@@ -91,6 +119,10 @@
"logistic_13x21x3_qu8",
"logistic_1x8x4x17_qi8",
"logistic_64_qi16"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"max_pool_2d": {
@@ -103,6 +135,10 @@
"max_pool_2d_1x4x8x19_qu8_st21_padVALID_kern22",
"max_pool_2d_1x8x4x17_qu8_st21_padVALID_kern21",
"max_pool_2d_1x32x32x8_qi8_st12_padVALID_kern22"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"minimum": {
@@ -112,12 +148,20 @@
"tests": [
"minimum_1x4x4x4_i32",
"minimum_64_i32"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"mul": {
"tests": [
"mul_13x21x3_qi16",
"mul_1x4x8x19_qu8"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"pack": {
@@ -129,6 +173,10 @@
"pack_14x19_i32_axis1",
"pack_1x7x7x9_i32_axis2",
"pack_1x8x4x17_i32_axis4"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"reshape": {
@@ -136,12 +184,20 @@
"reshape_14x19_i32_rank1",
"reshape_13x21x3_i32_rank3",
"reshape_1x32x32x8_i32_rank2"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"resize_bilinear": {
"tests": [
"resize_bilinear_1x32x32x8_qi16",
"resize_bilinear_1x4x4x4_qi8"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"resize_nearest_neighbor": {
@@ -151,6 +207,10 @@
"tests": [
"resize_nearest_neighbor_1x4x8x19_qu8",
"resize_nearest_neighbor_1x8x4x17_qi8"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"softmax": {
@@ -158,6 +218,10 @@
"softmax_1_qi8",
"softmax_14x19_qi16",
"softmax_1x32x32x8_qu8"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"split": {
@@ -166,6 +230,10 @@
"split_1x32x32x8_i32_split4_axis2",
"split_1x4x8x19_i32_splitv_axis3",
"split_64_i32_split2_axis0"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"strided_slice": {
@@ -173,6 +241,10 @@
"strided_slice_13x21x3_i32_perm0",
"strided_slice_14x19_i32_perm1",
"strided_slice_64_i32_perm3"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"tanh": {
@@ -180,6 +252,10 @@
"tanh_13x21x3_qu8",
"tanh_14x19_qi8",
"tanh_1x8x4x17_qi16"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
}
}
\ No newline at end of file
diff --git a/verif/conformance/tosa_base_profile_ops_info.json b/verif/conformance/tosa_base_profile_ops_info.json
index 79e50ff..b9f392c 100644
--- a/verif/conformance/tosa_base_profile_ops_info.json
+++ b/verif/conformance/tosa_base_profile_ops_info.json
@@ -37,6 +37,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"add": {
@@ -77,6 +81,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"argmax": {
@@ -125,6 +133,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"arithmetic_right_shift": {
@@ -162,6 +174,10 @@
"shape",
"type",
"rounding"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"avg_pool2d": {
@@ -198,6 +214,10 @@
"shape",
"type",
"pad"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"bitwise_and": {
@@ -236,6 +256,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"bitwise_not": {
@@ -274,6 +298,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"bitwise_or": {
@@ -312,6 +340,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"bitwise_xor": {
@@ -350,6 +382,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"cast": {
@@ -412,6 +448,10 @@
"shape",
"type",
"output_type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"clamp": {
@@ -448,6 +488,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"clz": {
@@ -488,6 +532,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"concat": {
@@ -546,6 +594,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"cond_if": {
@@ -573,6 +625,10 @@
"variant",
"shape",
"cond"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"const": {
@@ -624,6 +680,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"conv2d": {
@@ -665,6 +725,10 @@
"shape",
"type",
"pad"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"conv3d": {
@@ -711,6 +775,10 @@
"type",
"pad",
"stride"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"depthwise_conv2d": {
@@ -752,6 +820,10 @@
"shape",
"type",
"pad"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"equal": {
@@ -792,6 +864,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"fully_connected": {
@@ -828,6 +904,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"gather": {
@@ -856,6 +936,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"greater": {
@@ -902,6 +986,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"greater_equal": {
@@ -942,6 +1030,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"identity": {
@@ -995,6 +1087,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"intdiv": {
@@ -1035,6 +1131,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"logical_and": {
@@ -1069,6 +1169,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"logical_left_shift": {
@@ -1117,6 +1221,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"logical_not": {
@@ -1151,6 +1259,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"logical_or": {
@@ -1185,6 +1297,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"logical_right_shift": {
@@ -1233,6 +1349,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"logical_xor": {
@@ -1267,6 +1387,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"matmul": {
@@ -1303,6 +1427,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"maximum": {
@@ -1343,6 +1471,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"max_pool2d": {
@@ -1381,6 +1513,10 @@
"shape",
"type",
"pad"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"minimum": {
@@ -1421,6 +1557,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"mul": {
@@ -1471,6 +1611,10 @@
"type",
"perm",
"shift"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"negate": {
@@ -1519,6 +1663,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"pad": {
@@ -1585,6 +1733,10 @@
"type": "i8",
"pad": "pad00"
}
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"reduce_all": {
@@ -1606,6 +1758,10 @@
"shape",
"type",
"axis"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"reduce_any": {
@@ -1627,6 +1783,10 @@
"shape",
"type",
"axis"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"reduce_max": {
@@ -1656,6 +1816,10 @@
"shape",
"type",
"axis"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"reduce_min": {
@@ -1685,6 +1849,10 @@
"shape",
"type",
"axis"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"reduce_sum": {
@@ -1710,6 +1878,10 @@
"shape",
"type",
"axis"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"rescale": {
@@ -1771,6 +1943,10 @@
"shape",
"type",
"output_type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"reshape": {
@@ -1805,6 +1981,10 @@
"type",
"rank",
"perm"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"resize": {
@@ -1827,6 +2007,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"reverse": {
@@ -1857,6 +2041,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"scatter": {
@@ -1885,6 +2073,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"select": {
@@ -1939,6 +2131,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"slice": {
@@ -1996,6 +2192,10 @@
"shape",
"type",
"perm"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"sub": {
@@ -2036,6 +2236,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"table": {
@@ -2072,6 +2276,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"tile": {
@@ -2098,6 +2306,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"transpose": {
@@ -2151,6 +2363,10 @@
"shape",
"type",
"perm"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"transpose_conv2d": {
@@ -2193,6 +2409,10 @@
"type",
"stride",
"pad"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
},
"while_loop": {
@@ -2215,6 +2435,10 @@
"permutes": [
"shape",
"type"
+ ],
+ "profile": [
+ "tosa-bi",
+ "tosa-mi"
]
}
}
\ No newline at end of file
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index 836c639..c057b73 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -31,7 +31,7 @@
# Configuration for each TOSA profile
PROFILE_OPS_INFO = {
- "base": {
+ "tosa-bi": {
"operator_test_params": "tosa_base_profile_ops_info.json",
"framework_tests": "tosa_base_profile_framework_ops_info.json",
"exclude_types": ["float"],
@@ -219,6 +219,7 @@
operator,
op_build_dir,
output_dir,
+ profiles,
tests=None,
group=None,
trim_op_subdir=False,
@@ -230,6 +231,8 @@
output_dir = output_dir / group
ref_cmd_base = ["--ref-model-directory", str(ref_model_dir)]
+ for profile in profiles:
+ ref_cmd_base.extend(["--profile", profile])
if args.framework_schema:
ref_cmd_base.extend(["--framework-schema", str(args.framework_schema)])
ref_cmd_base.append("--output-directory")
@@ -388,6 +391,15 @@
def parse_args(argv=None):
"""Parse the arguments."""
parser = argparse.ArgumentParser()
+ profiles = list(PROFILE_OPS_INFO.keys())
+ parser.add_argument(
+ "--profile",
+ dest="profile",
+ choices=profiles,
+ default=profiles[0],
+ type=str,
+ help=f"TOSA profile (default is {profiles[0]})",
+ )
parser.add_argument(
"--operators",
type=str,
@@ -395,19 +407,62 @@
help="The operator(s) to create tests for, if not supplied all tests will be created",
)
parser.add_argument(
+ "--unit-tests",
+ dest="unit_tests",
+ choices=["operator", "framework", "both"],
+ default="operator",
+ type=str,
+ help="Which unit tests are produced (default is operator)",
+ )
+ parser.add_argument(
+ "--test-type",
+ dest="test_type",
+ choices=["positive", "negative", "both"],
+ default="both",
+ type=str,
+ help="Type of tests produced (default is both)",
+ )
+ parser.add_argument(
"--ref-model-directory",
dest="ref_model_dir",
type=Path,
required=True,
help="Reference Model directory (must be pre-built)",
)
+ parser.add_argument(
+ "--framework-tests-directory",
+ dest="framework_tests_dir",
+ type=Path,
+ default=Path.cwd() / "tests",
+ help="The pre-built framework tests directory (default is tests)",
+ )
+ parser.add_argument(
+ "--framework-schema",
+ dest="framework_schema",
+ type=Path,
+ help="Framework flatbuffers schema needed to convert framework models",
+ )
+ parser.add_argument(
+ "--build-directory",
+ dest="build_dir",
+ type=Path,
+ default=Path.cwd() / "conformance_build",
+ help="Temporary build directory for files created during this process (default is conformance_build)",
+ )
+ parser.add_argument(
+ "--output-directory",
+ dest="output_dir",
+ type=Path,
+ default=Path.cwd() / "conformance",
+ help="Output directory (default is conformance)",
+ )
script_dir = Path(__file__).parent.absolute()
parser.add_argument(
"--test-param-json-directory",
dest="param_json_dir",
type=Path,
default=script_dir,
- help="Test parameters (ops info) JSON file directory",
+ help=f"Test parameters (ops info) JSON file directory (default is {script_dir})",
)
parser.add_argument(
"--convert-all-tests",
@@ -425,20 +480,6 @@
help="Prints output of running sh commands",
)
parser.add_argument(
- "--build-directory",
- dest="build_dir",
- type=Path,
- default=Path.cwd() / "conformance_build",
- help="Temporary build directory for files created during this process (default is conformance_build)",
- )
- parser.add_argument(
- "--output-directory",
- dest="output_dir",
- type=Path,
- default=Path.cwd() / "conformance",
- help="Output directory (default is conformance)",
- )
- parser.add_argument(
"-j",
dest="num_cores",
type=int,
@@ -452,44 +493,6 @@
default=0,
help="Verbosity (can be used multiple times for more details)",
)
- parser.add_argument(
- "--unit-tests",
- dest="unit_tests",
- choices=["operator", "framework", "both"],
- default="operator",
- type=str,
- help="Which unit tests are produced: operator, framework, or both",
- )
- parser.add_argument(
- "--test-type",
- dest="test_type",
- choices=["positive", "negative", "both"],
- default="both",
- type=str,
- help="Type of tests produced: positive, negative, or both",
- )
- profiles = list(PROFILE_OPS_INFO.keys())
- parser.add_argument(
- "--profile",
- dest="profile",
- choices=profiles,
- default=profiles[0],
- type=str,
- help="TOSA profile",
- )
- parser.add_argument(
- "--framework-tests-directory",
- dest="framework_tests_dir",
- type=Path,
- default=Path.cwd() / "tests",
- help="The pre-built framework tests directory (default is tests)",
- )
- parser.add_argument(
- "--framework-schema",
- dest="framework_schema",
- type=Path,
- help="Framework flatbuffers schema needed to convert framework models",
- )
args = parser.parse_args(argv)
return args
@@ -567,6 +570,7 @@
logger.debug(f"Copying and renaming {op}")
framework_test_dir = copy_rename_framework_tests(args, op, test_picks)
+ profiles = test_picks[op]["profile"]
if args.convert_all_tests:
logger.debug("Running and converting all framework tests")
convert_tests(
@@ -574,6 +578,7 @@
op,
framework_test_dir,
root_output_dir,
+ profiles,
trim_op_subdir=True,
)
else:
@@ -585,6 +590,7 @@
op,
framework_test_dir,
root_output_dir,
+ profiles,
tests=framework_tests,
trim_op_subdir=True,
)
@@ -629,11 +635,17 @@
operator_group = test_params[op]["group"]
root_output_dir = args.output_dir / "operators"
+ profiles = test_params[op]["profile"]
if args.convert_all_tests:
logger.debug(f"Running and converting all {op} tests")
generate_results(args, op, op_build_dir)
output_dir = convert_tests(
- args, op, op_build_dir, root_output_dir, group=operator_group
+ args,
+ op,
+ op_build_dir,
+ root_output_dir,
+ profiles,
+ group=operator_group,
)
else:
if args.test_type in ["positive", "both"]:
@@ -646,6 +658,7 @@
op,
op_build_dir,
root_output_dir,
+ profiles,
tests=tests_gen2,
group=operator_group,
)
@@ -661,6 +674,7 @@
op,
op_build_dir,
root_output_dir,
+ profiles,
tests=negative_tests,
group=operator_group,
)
diff --git a/verif/runner/tosa_test_runner.py b/verif/runner/tosa_test_runner.py
index d653a94..65931d8 100644
--- a/verif/runner/tosa_test_runner.py
+++ b/verif/runner/tosa_test_runner.py
@@ -76,13 +76,17 @@
self.descFile = str(descFilePath)
def skipTest(self):
- """Check if the test is skipped due to test type selection."""
+ """Check if the test is skipped due to test type or profile selection."""
expectedFailure = self.testDesc["expected_failure"]
if self.args.test_type == "negative" and not expectedFailure:
- return True
+ return True, "non-negative type"
elif self.args.test_type == "positive" and expectedFailure:
- return True
- return False
+ return True, "non-positive type"
+ if self.args.profile:
+ profile = self.testDesc["profile"] if "profile" in self.testDesc else []
+ if self.args.profile not in profile:
+ return True, "non-{} profile".format(self.args.profile)
+ return False, ""
def runTestGraph(self):
"""Override with function that calls system under test."""
diff --git a/verif/runner/tosa_verif_run_tests.py b/verif/runner/tosa_verif_run_tests.py
index b400d76..77394cc 100644
--- a/verif/runner/tosa_verif_run_tests.py
+++ b/verif/runner/tosa_verif_run_tests.py
@@ -117,7 +117,7 @@
type=str,
default="both",
choices=["positive", "negative", "both"],
- help="Filter tests based on expected failure status (positive, negative or both)",
+ help="Filter tests based on expected failure status",
)
parser.add_argument(
"--no-color",
@@ -126,6 +126,13 @@
action="store_true",
help="Disable color output",
)
+ parser.add_argument(
+ "--profile",
+ dest="profile",
+ type=str,
+ choices=["tosa-bi", "tosa-mi"],
+ help="Filter tests based on profile",
+ )
args = parser.parse_args(argv)
@@ -171,8 +178,9 @@
runnerName = runnerModule.__name__
runner = runnerModule.TosaSUTRunner(args, runnerArgs, test)
- if runner.skipTest():
- msg = "Skipping non-{} test".format(args.test_type)
+ skip, reason = runner.skipTest()
+ if skip:
+ msg = "Skipping {} test".format(reason)
print("{} {}".format(msg, test))
rc = TosaTestRunner.Result.SKIPPED
else: