Fixes for conformance generator tags and test selection
Incorrectly used "tag" in test.desc instead of "tags"
Incorrectly selected all tests rather than just all positive
tests depending on operator selection option.
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Ib257bc57accf745beedfa4a3b2ece9539c0972c2
diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py
index f915070..555cb50 100755
--- a/scripts/convert2conformance/convert2conformance.py
+++ b/scripts/convert2conformance/convert2conformance.py
@@ -83,10 +83,10 @@
)
parser.add_argument(
"--tag",
- dest="tag",
+ dest="tags",
action="append",
type=str,
- help="Optional string tag mark this test with. May be repeated",
+ help="Optional string tag to mark this test with. May be repeated",
)
parser.add_argument(
"--strict",
@@ -218,7 +218,7 @@
# Add tags (if any)
if tags is not None:
- test_desc["tag"] = tags
+ test_desc["tags"] = tags
return test_desc
@@ -355,14 +355,18 @@
output_dir=args.output_dir,
create_result=(not args.lazy_data_generation),
profiles=args.profile,
- tags=args.tag,
+ tags=args.tags,
)
if not test_desc:
# Error from conversion/update
return 1
# Validate the desc.json schema
- TestDescSchemaValidator().validate_config(test_desc)
+ try:
+ TestDescSchemaValidator().validate_config(test_desc)
+ except Exception as e:
+ logger.error(e)
+ return 1
# Output new desc.json
new_desc_filename = args.output_dir / NAME_DESC_FILENAME
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index c2ea4ec..236f729 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -805,10 +805,7 @@
# Selection criteria
selection_config = test_params[op]["selection"][selector_name]
- if args.convert_all_tests or (
- "all" in selection_config
- and selection_config["all"] == "true"
- ):
+ if args.convert_all_tests:
logger.debug(f"Running and converting all {op} tests")
generate_results(
args, profile, op, op_build_dir, supports=supports
@@ -819,16 +816,31 @@
f"Running and converting selection of {op} tests"
)
if test_type in ["positive", "both"]:
- tests_gen, tests_gen2 = tee(
- get_op_tests_selection(
- args,
- profile,
- op,
- op_build_dir,
- selection_config,
- ignore_missing=ignore_missing,
+ if (
+ "all" in selection_config
+ and selection_config["all"] == "true"
+ ):
+ # Just get all the positive tests
+ tests_gen, tests_gen2 = tee(
+ _get_all_tests_list(
+ profile,
+ op_build_dir,
+ op,
+ exclude_negative_tests=True,
+ )
)
- )
+ else:
+ # Get a selection of positive tests
+ tests_gen, tests_gen2 = tee(
+ get_op_tests_selection(
+ args,
+ profile,
+ op,
+ op_build_dir,
+ selection_config,
+ ignore_missing=ignore_missing,
+ )
+ )
generate_results(
args,
profile,