Fix missing/broken ERROR_IF tests
Fix CONV2D WrongOutputType FP32 & Pad/Stride/DilationSmallerZero issues.
Fix PAD WrongInputType.
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I57fc57c43e63685e05bf5e3d562c3167411fd57b
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index 5402c21..3b7c2df 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -82,7 +82,9 @@
)
if args.capture_output:
+ stderr = rc.stderr.decode("utf-8")
stdout = rc.stdout.decode("utf-8")
+ logger.info(f"stderr: \n{stderr}")
logger.info(f"stdout: \n{stdout}")
if rc.returncode != 0:
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index ffa3683..4878708 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -1976,6 +1976,9 @@
sparsity += 1
return sparsity
+ # Maximum number of error_if variants to produce
+ MAX_CONV_ERROR_IFS = 3
+
@staticmethod
def agConv(testGen, rng, opName, shapeList, dtypes, error_name=None):
# Used by CONV2D, CONV3D and DEPTHWISE_CONV2D
@@ -2017,17 +2020,60 @@
k_size *= ifm_shape[-1]
if not testGen.args.level8k:
- # Generate comprehensive argument lists
- # - except for named errors, which use specific invalid value(s)
- if error_name == ErrorIf.PadSmallerZero:
- p_vals = [rng.choice(range(-5, 0))]
+ if error_name in (
+ ErrorIf.PadSmallerZero,
+ ErrorIf.StrideSmallerOne,
+ ErrorIf.DilationSmallerOne,
+ ):
+ # Use specific invalid value(s)
+ if error_name == ErrorIf.PadSmallerZero:
+ # Create negative paddings but with positive opposite paddings
+ neg_pad = rng.choice(range(-5, 0))
+ p_vals = [neg_pad, abs(neg_pad)]
+ else:
+ p_vals = [0, 0]
+ if error_name == ErrorIf.StrideSmallerOne:
+ # Can't use stride=0, as it is used to derive output shape, as a divisor
+ s_vals = [rng.choice(range(-5, 0))]
+ else:
+ s_vals = [1]
+ if error_name == ErrorIf.DilationSmallerOne:
+ d_vals = [rng.choice(range(-5, 1))]
+ else:
+ d_vals = [1]
+ p = p_vals * k_rank
+ s = s_vals * k_rank
+ d = d_vals * k_rank
+
+ # Fix values to produce valid error_if
+ for index in range(k_rank):
+ pad_offset = index * 2
+ fixed = False
+ while not fixed:
+ partial = (
+ ifm_shape[index + 1]
+ - 1
+ + p[pad_offset]
+ + p[pad_offset + 1]
+ - (k_shape[index] - 1) * d[index]
+ )
+ remainder = partial % s[index]
+ if partial <= 0:
+ p[pad_offset + 1] += abs(partial) + 1
+ elif remainder:
+ # Stride will be negative for StrideSmallerOne
+ assert remainder < 0
+ p[pad_offset + 1] += abs(remainder)
+ else:
+ fixed = True
+ paddings = {tuple(p)}
+ strides = {tuple(s)}
+ dilations = {tuple(d)}
+ logger.debug(f"agConv: error pad={p} stride={s} dilation={d}")
else:
+ # Generate comprehensive argument lists
p_vals = [x for x in range(0, testGen.args.max_conv_padding + 1)]
- paddings = {x for x in itertools.product(*([p_vals] * k_rank * 2))}
- if error_name == ErrorIf.StrideSmallerOne:
- # Can't use stride=0, as it is used to derive output shape, as a divisor
- s_vals = [rng.choice(range(-5, 0))]
- else:
+ paddings = {x for x in itertools.product(*([p_vals] * k_rank * 2))}
# Stride must be greater than 1 to force non-integer error
startStride = (
1 if error_name != ErrorIf.ConvOutputShapeNonInteger else 2
@@ -2035,12 +2081,10 @@
s_vals = [
x for x in range(startStride, testGen.args.max_conv_stride + 1)
]
- strides = {x for x in itertools.product(*([s_vals] * k_rank))}
- if error_name == ErrorIf.DilationSmallerOne:
- d_vals = [rng.choice(range(-5, 1))]
- else:
d_vals = [x for x in range(1, testGen.args.max_conv_dilation + 1)]
- dilations = {x for x in itertools.product(*([d_vals] * k_rank))}
+
+ strides = {x for x in itertools.product(*([s_vals] * k_rank))}
+ dilations = {x for x in itertools.product(*([d_vals] * k_rank))}
if not error_name and testGen.args.oversize:
# add some oversize argument values
@@ -2064,12 +2108,15 @@
)
max_dim_size = None
- # There are too many parameter combinations, so generate them sparsely,
- # very sparse for negative tests
- sparsity_factor = 2 if error_name else 120
- sparsity = TosaArgGen._calculate_sparsity(
- len(paddings) * len(strides) * len(dilations), sparsity_factor
- )
+ if error_name:
+ # Cycle through all error_if tests but we only keep the first few
+ sparsity = 1
+ else:
+ # There are too many parameter combinations, so generate them sparsely,
+ sparsity_factor = 120
+ sparsity = TosaArgGen._calculate_sparsity(
+ len(paddings) * len(strides) * len(dilations), sparsity_factor
+ )
else:
# Only test 8k levels boundaries
bigStride = testGen.TOSA_8K_LEVEL_MAX_STRIDE
@@ -2114,13 +2161,15 @@
# Currently allow all combinations that are reasonable size
sparsity = 1
+ more_tests = True
n = 0
for a in accum_dtypes:
for s in sorted(list(strides)):
for p in sorted(list(paddings)):
for d in sorted(list(dilations)):
if (
- n % sparsity == 0
+ more_tests
+ and n % sparsity == 0
# the padded shape must exceed the dilation * kernel to get a positive
# sized output shape
and (ifm_shape[1] - 1 + p[0] + p[1])
@@ -2199,6 +2248,15 @@
args_dict,
)
)
+ if (
+ error_name
+ and len(arg_list) >= TosaArgGen.MAX_CONV_ERROR_IFS
+ ):
+ # Found enough errors
+ logger.debug(
+ f"Skipping creating more conv error tests for {error_name}"
+ )
+ more_tests = False
n += 1
arg_list = TosaArgGen._add_data_generators(
@@ -2482,7 +2540,9 @@
pad_const_int = 0
pad_const_fp = rng.randNumberDType(dtype)
else:
- return []
+ assert error_name == ErrorIf.WrongInputType
+ pad_const_int = 0
+ pad_const_fp = 0
list_shape_pad_values = list(shape_pad_values)
# If we are producing tests for rank 6 or greater use sparsity
@@ -2523,7 +2583,9 @@
arg_list.append((name, args_dict))
if error_name == ErrorIf.PadSmallerZero and len(arg_list) == 0:
- logger.info(f"No ErrorIf test created for input shape: {shapeList[0]}")
+ logger.debug(
+ f"agPad: No PadSmallerZero ErrorIf test created for input shape: {shapeList[0]}"
+ )
arg_list = TosaArgGen._add_data_generators(
testGen,
@@ -3106,8 +3168,8 @@
# Find new shapes up to the number of permutations asked for
# This code is NOT fast. Fortunately, the numbers are fairly small.
for p in range(testGen.args.num_rand_permutations):
- # Rank from 1 to TOSA_TENSOR_MAX_RANK
- newRank = rng.randInt(1, (testGen.TOSA_TENSOR_MAX_RANK + 1))
+ # Rank from 1 to MAX_TENSOR_RANK
+ newRank = rng.randInt(1, (gtu.MAX_TENSOR_RANK + 1))
if len(factors) < newRank:
continue
diff --git a/verif/generator/tosa_error_if.py b/verif/generator/tosa_error_if.py
index 916b4f9..1b6b044 100644
--- a/verif/generator/tosa_error_if.py
+++ b/verif/generator/tosa_error_if.py
@@ -3,11 +3,8 @@
import logging
import math
+import generator.tosa_utils as gtu
import numpy as np
-from generator.tosa_utils import MAX_RESIZE_DIMENSION
-from generator.tosa_utils import product
-from generator.tosa_utils import usableDTypes
-from generator.tosa_utils import valueToName
from tosa.DType import DType
from tosa.Op import Op
from tosa.ResizeMode import ResizeMode
@@ -287,7 +284,7 @@
max_dim and max_items.
"""
new_shape = [min(d, max_dim) for d in shape] if max(shape) > max_dim else shape
- while product(new_shape) > max_items:
+ while gtu.product(new_shape) > max_items:
new_shape = [max(d - 1, 1) for d in new_shape]
return new_shape
@@ -389,12 +386,12 @@
serializer.setExpectedReturnCode(2, True, desc=error_reason)
elif error_result: # and not expected_result
logger.error(
- f"Unexpected ERROR_IF: Op: {valueToName(Op, kwargs['op']['op'])}"
+ f"Unexpected ERROR_IF: Op: {gtu.valueToName(Op, kwargs['op']['op'])}"
f" Expected: {error_name}, Got: {validator_name}"
)
elif not expected_result: # and not error_result
logger.error(
- f"Missed ERROR_IF: Op: {valueToName(Op, kwargs['op']['op'])}"
+ f"Missed ERROR_IF: Op: {gtu.valueToName(Op, kwargs['op']['op'])}"
f" Expected: {error_name}"
)
@@ -402,7 +399,7 @@
for k, v in sorted(kwargs.items()):
if k != "op":
if k.endswith("dtype"):
- v = valueToName(DType, v)
+ v = gtu.valueToName(DType, v)
logger.error(f" {k} = {v}")
return overall_result
@@ -417,7 +414,8 @@
allowed_input_dtypes = {
t[0] if isinstance(t, list) else t for t in input_dtypes
}
- wrong_input_dtypes = list(usableDTypes(excludes=allowed_input_dtypes))
+ wrong_input_dtypes = list(gtu.usableDTypes(excludes=allowed_input_dtypes))
+ assert len(wrong_input_dtypes) > 0
# Turn the wrong dtypes into required list of types
if op["op"] in [
@@ -682,7 +680,8 @@
@staticmethod
def evWrongRank(check=False, **kwargs):
- all_ranks = (1, 2, 3, 4, 5)
+ # From 1 to MAX_TENSOR_RANK+1 inclusively
+ all_ranks = tuple(range(1, gtu.MAX_TENSOR_RANK + 2))
# Make a list of incorrect ranks
assert "op" in kwargs
@@ -784,16 +783,16 @@
"shape": [[1, 16584, 5, 1], [1, 2, 16499, 4]],
}
error_result = False
- error_reason = f"At least one maximum dimension is greater than or equal to {MAX_RESIZE_DIMENSION}"
+ error_reason = f"At least one maximum dimension is greater than or equal to {gtu.MAX_RESIZE_DIMENSION}"
if check:
input_shape = kwargs["input_shape"]
output_shape = kwargs["output_shape"]
if (
- (input_shape[1] >= MAX_RESIZE_DIMENSION)
- or (input_shape[2] >= MAX_RESIZE_DIMENSION)
- or (output_shape[1] >= MAX_RESIZE_DIMENSION)
- or (output_shape[2] >= MAX_RESIZE_DIMENSION)
+ (input_shape[1] >= gtu.MAX_RESIZE_DIMENSION)
+ or (input_shape[2] >= gtu.MAX_RESIZE_DIMENSION)
+ or (output_shape[1] >= gtu.MAX_RESIZE_DIMENSION)
+ or (output_shape[2] >= gtu.MAX_RESIZE_DIMENSION)
):
error_result = True
@@ -1075,8 +1074,8 @@
if (
params_valid
- and max(output_shape) < MAX_RESIZE_DIMENSION
- and max(input_shape) < MAX_RESIZE_DIMENSION
+ and max(output_shape) < gtu.MAX_RESIZE_DIMENSION
+ and max(input_shape) < gtu.MAX_RESIZE_DIMENSION
):
output_y = (
(input_shape[1] - 1) * scale[0] - offset[0] + border[0]
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index c867070..28b3d28 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -36,9 +36,7 @@
class TosaTestGen:
- # Maximum rank of tensor supported by test generator.
# This currently matches the 8K level defined in the specification.
- TOSA_TENSOR_MAX_RANK = 6
TOSA_8K_LEVEL_MAX_SCALE = 64
TOSA_8K_LEVEL_MAX_KERNEL = 8192
TOSA_8K_LEVEL_MAX_STRIDE = 8192
@@ -2941,8 +2939,10 @@
testList = []
if testType == "negative" and "error_if_validators" in op:
error_if_validators = op["error_if_validators"]
+ num_error_types_created = 0
else:
error_if_validators = [None]
+ num_error_types_created = None
for validator in error_if_validators:
if validator is not None:
@@ -3020,6 +3020,21 @@
testList.append(
(opName, testStr, t, error_name, shapeList, args)
)
+ if error_name is not None:
+ # Check the last test is of the error we wanted
+ if len(testList) == 0 or testList[-1][3] != error_name:
+ if self.args.level8k:
+ logger.info(f"Missing {error_name} tests due to level8k mode")
+ else:
+ logger.error(f"ERROR: Failed to create any {error_name} tests")
+ logger.debug(
+ "Last test created: {}".format(
+ testList[-1] if testList else None
+ )
+ )
+ else:
+ # Successfully created at least one ERRROR_IF test
+ num_error_types_created += 1
if testType == "positive":
# Remove tests which are expected to fail but don't correlate to a ERROR_IF statement
@@ -3039,6 +3054,15 @@
if not remove_test:
clean_testList.append(test)
testList = clean_testList
+ else:
+ if num_error_types_created is not None and not self.args.level8k:
+ remaining_error_types = (
+ len(error_if_validators) - num_error_types_created
+ )
+ if remaining_error_types:
+ raise Exception(
+ f"Failed to create {remaining_error_types} error types for {opName}"
+ )
return testList
@@ -3141,9 +3165,11 @@
if compliance:
tensMeta["compliance"] = compliance
self.serialize("test", tensMeta)
+ return True
else:
# The test is not valid
logger.error(f"Invalid ERROR_IF test created: {opName} {testStr}")
+ return False
def createDynamicOpLists(self):
@@ -3305,7 +3331,7 @@
[DType.FP8E5M2, DType.FP8E5M2, DType.FP16],
]
- DEFAULT_RANK_RANGE = (1, TOSA_TENSOR_MAX_RANK)
+ DEFAULT_RANK_RANGE = (1, gtu.MAX_TENSOR_RANK)
TOSA_OP_LIST = {
# Tensor operators
diff --git a/verif/generator/tosa_test_select.py b/verif/generator/tosa_test_select.py
index c6e1d0d..904d90f 100644
--- a/verif/generator/tosa_test_select.py
+++ b/verif/generator/tosa_test_select.py
@@ -185,7 +185,7 @@
# Add a test to this op group and set up the permutations/group for it
assert test.opName.startswith(self.opName)
if str(test) in self.testStrings:
- logger.info(f"Skipping duplicate test: {str(test)}")
+ logger.debug(f"Skipping duplicate test: {str(test)}")
return
self.tests.append(test)
diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py
index 4a4f6bb..a8e321e 100644
--- a/verif/generator/tosa_utils.py
+++ b/verif/generator/tosa_utils.py
@@ -10,6 +10,9 @@
# Maximum dimension size for output and inputs for RESIZE
MAX_RESIZE_DIMENSION = 16384
+# Maximum rank of tensor supported by test generator.
+MAX_TENSOR_RANK = 6
+
# Data type information dictionary
# - str: filename abbreviation
# - width: number of bytes needed for type
diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py
index 83c06d7..0839cec 100644
--- a/verif/generator/tosa_verif_build_tests.py
+++ b/verif/generator/tosa_verif_build_tests.py
@@ -428,6 +428,9 @@
logger.error(f"INTERNAL ERROR: Failure creating test output for {opName}")
raise e
+ if results.count(False):
+ raise Exception(f"Failed to create {results.count(False)} tests")
+
if not args.list_tests:
print(f"Done creating {len(results)} tests")
return 0