Main Compliance testing support for ADD, SUB, MINIMUM and MAXIMUM
Added main inference compliance mode data generation and verification.
Improved error reporting in tosa_verif_build_tests.
Fixed pseudo random invalid range check.
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Ib538b5999444d67704ebc322be3c7becbe5ad206
diff --git a/reference_model/src/generate/generate_pseudo_random.cc b/reference_model/src/generate/generate_pseudo_random.cc
index f234796..78013eb 100644
--- a/reference_model/src/generate/generate_pseudo_random.cc
+++ b/reference_model/src/generate/generate_pseudo_random.cc
@@ -126,9 +126,10 @@
WARNING("[Generator][PR] Unknown operator.");
return false;
}
- if (cfg.pseudoRandomInfo.range.size() != 0 || cfg.pseudoRandomInfo.range.size() != 2)
+ if (cfg.pseudoRandomInfo.range.size() != 0 && cfg.pseudoRandomInfo.range.size() != 2)
{
- WARNING("[Generator][PR] Invalid range.");
+ WARNING("[Generator][PR] Invalid range");
+ return false;
}
switch (cfg.dataType)
diff --git a/reference_model/src/generate/generate_utils.cc b/reference_model/src/generate/generate_utils.cc
index ae6dfcb..8ae889a 100644
--- a/reference_model/src/generate/generate_utils.cc
+++ b/reference_model/src/generate/generate_utils.cc
@@ -38,11 +38,15 @@
NLOHMANN_JSON_SERIALIZE_ENUM(Op,
{
{ Op::Op_UNKNOWN, "UNKNOWN" },
+ { Op::Op_ADD, "ADD" },
{ Op::Op_CONV2D, "CONV2D" },
{ Op::Op_MATMUL, "MATMUL" },
+ { Op::Op_MAXIMUM, "MAXIMUM" },
{ Op::Op_MAX_POOL2D, "MAX_POOL2D" },
+ { Op::Op_MINIMUM, "MINIMUM" },
{ Op::Op_MUL, "MUL" },
{ Op::Op_PAD, "PAD" },
+ { Op::Op_SUB, "SUB" },
})
} // namespace tosa
diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json
index 4256bfb..254f5e7 100644
--- a/verif/conformance/tosa_main_profile_ops_info.json
+++ b/verif/conformance/tosa_main_profile_ops_info.json
@@ -69,6 +69,7 @@
"profile": [
"tosa-mi"
],
+ "support_for": [ "lazy_data_gen" ],
"generation": {
"standard": {
"generator_args": [
@@ -80,7 +81,7 @@
"--target-dtype",
"bf16",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--tensor-dim-range",
"16,64",
"--target-rank",
@@ -94,7 +95,7 @@
"--target-dtype",
"fp32",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--tensor-dim-range",
"1,16",
"--target-rank",
@@ -106,7 +107,7 @@
"--target-dtype",
"fp16",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--target-shape",
"2,1,1,65532",
"--target-shape",
@@ -2182,6 +2183,7 @@
"profile": [
"tosa-mi"
],
+ "support_for": [ "lazy_data_gen" ],
"generation": {
"standard": {
"generator_args": [
@@ -2193,7 +2195,7 @@
"--target-dtype",
"bf16",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--tensor-dim-range",
"16,64",
"--target-rank",
@@ -2211,7 +2213,7 @@
"--target-dtype",
"bf16",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--tensor-dim-range",
"1,16",
"--target-rank",
@@ -2223,7 +2225,7 @@
"--target-dtype",
"fp32",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--target-shape",
"65531,1,2,1",
"--target-shape",
@@ -2247,6 +2249,7 @@
"profile": [
"tosa-mi"
],
+ "support_for": [ "lazy_data_gen" ],
"generation": {
"standard": {
"generator_args": [
@@ -2258,7 +2261,7 @@
"--target-dtype",
"bf16",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--tensor-dim-range",
"16,64",
"--target-rank",
@@ -2276,7 +2279,7 @@
"--target-dtype",
"bf16",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--tensor-dim-range",
"1,16",
"--target-rank",
@@ -2288,7 +2291,7 @@
"--target-dtype",
"fp32",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--target-shape",
"1,65531,2,1",
"--target-shape",
@@ -2775,6 +2778,7 @@
"profile": [
"tosa-mi"
],
+ "support_for": [ "lazy_data_gen" ],
"generation": {
"standard": {
"generator_args": [
@@ -2786,7 +2790,7 @@
"--target-dtype",
"bf16",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--tensor-dim-range",
"16,64",
"--target-rank",
@@ -2800,7 +2804,7 @@
"--target-dtype",
"bf16",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--tensor-dim-range",
"1,16",
"--target-rank",
@@ -2812,7 +2816,7 @@
"--target-dtype",
"fp32",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--target-shape",
"1,1,65533,3",
"--target-shape",
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index 94b7172..1be243c 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -636,6 +636,19 @@
}
@staticmethod
+ def _get_data_range(testGen, dtype, highValueLookup):
+ if dtype in highValueLookup:
+ data_range = testGen.getDTypeRange(dtype, high_inclusive=True)
+ high_val = highValueLookup[dtype]
+ # Set the values to something that won't produce infinity whilst
+ # respecting the default ranges if less than the high value
+ return [
+ max(-high_val, data_range[0]),
+ min(high_val, data_range[1]),
+ ]
+ return None
+
+ @staticmethod
def tvgLazyGenDefault(
testGen, opName, dtypeList, shapeList, argsDict, error_name=None
):
@@ -777,16 +790,24 @@
testGen, op, dtypeList, shapeList, testArgs, error_name
)
+ # Set the data range to half the largest value
+ TVG_FLOAT_HIGH_VALUE_ADDSUB = {
+ DType.FP32: (TVG_FLOAT_HIGH_VALUE[DType.FP32] / 2),
+ DType.FP16: (TVG_FLOAT_HIGH_VALUE[DType.FP16] / 2),
+ DType.BF16: (TVG_FLOAT_HIGH_VALUE[DType.BF16] / 2),
+ }
+
@staticmethod
- def tvgAddSub(testGen, op, dtypeList, shapeList, testArgs, error_name=None):
+ def tvgAddSub(testGen, opName, dtypeList, shapeList, argsDict, error_name=None):
if dtypeList[0] == DType.INT32 and error_name is None:
- # Make sure the operation does not cause value saturation - where
+ # Make sure the integer operation does not cause value saturation - where
# the number wraps due to limited number of bits to store the answer
+ op = testGen.TOSA_OP_LIST[opName]
pCount, cCount = op["operands"]
assert (
pCount == 2 and cCount == 0
), "Op.ADD / Op.SUB must have 2 placeholders, 0 consts"
- placeholders = []
+ tens_ser_list = []
add = op["op"] == Op.ADD
a_arr = testGen.getRandTensor(shapeList[0], dtypeList[0])
b_arr = testGen.getRandTensor(shapeList[1], dtypeList[1])
@@ -833,17 +854,24 @@
), "Op.ADD / SUB dimension must be 1 or matching to be broadcastable"
b_unsat_arr = np.amax(b_unsat_arr, axis=axis, keepdims=True)
- placeholders.append(
+ tens_ser_list.append(
testGen.ser.addPlaceholder(shapeList[0], dtypeList[0], a_arr)
)
- placeholders.append(
+ tens_ser_list.append(
testGen.ser.addPlaceholder(shapeList[1], dtypeList[1], b_unsat_arr)
)
- return placeholders
+ return TosaTensorValuesGen.TVGInfo(tens_ser_list, None)
else:
- return TosaTensorValuesGen.tvgDefault(
- testGen, op, dtypeList, shapeList, testArgs, error_name
+ # ERROR_IF or floating point test
+ data_range = TosaTensorValuesGen._get_data_range(
+ testGen, dtypeList[0], TosaTensorValuesGen.TVG_FLOAT_HIGH_VALUE_ADDSUB
+ )
+ if data_range:
+ argsDict["data_range"] = data_range
+
+ return TosaTensorValuesGen.tvgLazyGenDefault(
+ testGen, opName, dtypeList, shapeList, argsDict, error_name
)
@staticmethod
@@ -923,14 +951,15 @@
)
@staticmethod
- def tvgIntDiv(testGen, op, dtypeList, shapeList, testArgs, error_name=None):
+ def tvgIntDiv(testGen, opName, dtypeList, shapeList, argsDict, error_name=None):
if error_name is None:
+ op = testGen.TOSA_OP_LIST[opName]
pCount, cCount = op["operands"]
assert (
pCount == 2 and cCount == 0
), "Op.INTDIV must have 2 placeholders, 0 consts"
- placeholders = []
+ tens_ser_list = []
# Two invalid cases for Op.INTDIV:
# 1. divisor == 0
@@ -947,17 +976,17 @@
break
- placeholders.append(
+ tens_ser_list.append(
testGen.ser.addPlaceholder(shapeList[0], dtypeList[0], dividend_arr)
)
- placeholders.append(
+ tens_ser_list.append(
testGen.ser.addPlaceholder(shapeList[1], dtypeList[1], divisor_arr)
)
- return placeholders
+ return TosaTensorValuesGen.TVGInfo(tens_ser_list, None)
else:
- return TosaTensorValuesGen.tvgDefault(
- testGen, op, dtypeList, shapeList, testArgs, error_name
+ return TosaTensorValuesGen.tvgLazyGenDefault(
+ testGen, opName, dtypeList, shapeList, argsDict, error_name
)
# Set the data range to the square root of the largest value
@@ -975,15 +1004,12 @@
DType.FP32,
):
# ERROR_IF or floating point test
- if dtypeList[0] in TosaTensorValuesGen.TVG_FLOAT_HIGH_VALUE_MUL:
- data_range = testGen.getDTypeRange(dtypeList[0], high_inclusive=True)
- high_val = TosaTensorValuesGen.TVG_FLOAT_HIGH_VALUE_MUL[dtypeList[0]]
- # Set the values to something that won't produce infinity whilst
- # respecting the default ranges if less than the high value
- argsDict["data_range"] = [
- max(-high_val, data_range[0]),
- min(high_val, data_range[1]),
- ]
+ data_range = TosaTensorValuesGen._get_data_range(
+ testGen, dtypeList[0], TosaTensorValuesGen.TVG_FLOAT_HIGH_VALUE_MUL
+ )
+ if data_range:
+ argsDict["data_range"] = data_range
+
return TosaTensorValuesGen.tvgLazyGenDefault(
testGen, opName, dtypeList, shapeList, argsDict, error_name
)
@@ -1075,22 +1101,25 @@
return tens
@staticmethod
- def tvgLogicalShift(testGen, op, dtypeList, shapeList, testArgs, error_name=None):
+ def tvgLogicalShift(
+ testGen, opName, dtypeList, shapeList, argsDict, error_name=None
+ ):
+ op = testGen.TOSA_OP_LIST[opName]
pCount, cCount = op["operands"]
assert (
pCount == 2 and cCount == 0
), "Op.LOGICAL_LEFT_SHIFT or Op.LOGICAL_RIGHT_SHIFT must have 2 placeholders, 0 consts"
values_arr = testGen.getRandTensor(shapeList[0], dtypeList[0])
shift_arr = np.int32(testGen.rng.integers(low=0, high=32, size=shapeList[1]))
- placeholders = []
- placeholders.append(
+ tens_ser_list = []
+ tens_ser_list.append(
testGen.ser.addPlaceholder(shapeList[0], dtypeList[0], values_arr)
)
- placeholders.append(
+ tens_ser_list.append(
testGen.ser.addPlaceholder(shapeList[1], dtypeList[1], shift_arr)
)
- return placeholders
+ return TosaTensorValuesGen.TVGInfo(tens_ser_list, None)
@staticmethod
def tvgEqual(testGen, op, dtypeList, shapeList, testArgs, error_name=None):
@@ -1218,7 +1247,15 @@
def agNone(testGen, opName, shapeList, dtype, error_name=None):
"""A trivial argument generator for operators that don't take any
non-tensor arguments"""
- return [("", [])]
+ arg_list = TosaArgGen._add_data_generators(
+ testGen,
+ opName,
+ dtype,
+ [("", {})],
+ error_name,
+ )
+ # Return list of tuples: (arg_str, args_dict)
+ return arg_list
@staticmethod
def agAxis(testGen, opName, shapeList, dtype, error_name=None):
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index 1995cbc..556a0d8 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -403,14 +403,18 @@
self.ser.addOperator(op["op"], input_list, output_list, attr)
return result_tens
- def build_binary_broadcast(self, op, a, b, validator_fcns, error_name=None):
- result_tens = OutputShaper.binaryBroadcastOp(
+ def build_binary_broadcast(
+ self, op, inputs, args_dict, validator_fcns, error_name=None, qinfo=None
+ ):
+ assert len(inputs) == 2
+ a, b = inputs
+ result_tensor = OutputShaper.binaryBroadcastOp(
self.ser, self.rng, a, b, error_name
)
# Invalidate Input/Output list for error if checks.
input_list = [a.name, b.name]
- output_list = [result_tens.name]
+ output_list = [result_tensor.name]
pCount, cCount = op["operands"]
num_operands = pCount + cCount
input_list, output_list = TosaErrorIfArgGen.eiInvalidateInputOutputList(
@@ -425,8 +429,8 @@
input1=a,
input2=b,
input_dtype=a.dtype,
- output_dtype=result_tens.dtype,
- result_tensors=[result_tens],
+ output_dtype=result_tensor.dtype,
+ result_tensors=[result_tensor],
input_list=input_list,
output_list=output_list,
num_operands=num_operands,
@@ -434,7 +438,16 @@
return None
self.ser.addOperator(op["op"], input_list, output_list)
- return result_tens
+
+ if op["op"] == Op.POW:
+ # TODO - add compliance support
+ compliance = None
+ else:
+ compliance = self.tensorComplianceMetaData(
+ op, a.dtype, args_dict, result_tensor, error_name
+ )
+
+ return TosaTestGen.BuildInfo(result_tensor, compliance)
def build_binary_nonbroadcast(self, op, a, b, validator_fcns=None, error_name=None):
result_tens = OutputShaper.binaryNonBroadcastOp(self.ser, a, b)
@@ -3208,7 +3221,7 @@
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
TosaTensorValuesGen.tvgAddSub,
- None,
+ TosaArgGen.agNone,
),
"types": TYPE_FI32,
"error_if_validators": (
@@ -3220,6 +3233,10 @@
TosaErrorValidator.evDimensionMismatch,
TosaErrorValidator.evBroadcastShapesMismatch,
),
+ "data_gen": {
+ "fp": (gtu.DataGenType.PSEUDO_RANDOM,),
+ },
+ "compliance": {"ulp": 0.5},
},
"arithmetic_right_shift": {
"op": Op.ARITHMETIC_RIGHT_SHIFT,
@@ -3247,8 +3264,8 @@
"build_fcn": (
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
- TosaTensorValuesGen.tvgDefault,
- None,
+ TosaTensorValuesGen.tvgLazyGenDefault,
+ TosaArgGen.agNone,
),
"types": TYPE_INT,
"error_if_validators": (
@@ -3267,8 +3284,8 @@
"build_fcn": (
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
- TosaTensorValuesGen.tvgDefault,
- None,
+ TosaTensorValuesGen.tvgLazyGenDefault,
+ TosaArgGen.agNone,
),
"types": TYPE_INT,
"error_if_validators": (
@@ -3287,8 +3304,8 @@
"build_fcn": (
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
- TosaTensorValuesGen.tvgDefault,
- None,
+ TosaTensorValuesGen.tvgLazyGenDefault,
+ TosaArgGen.agNone,
),
"types": TYPE_INT,
"error_if_validators": (
@@ -3308,7 +3325,7 @@
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
TosaTensorValuesGen.tvgIntDiv,
- None,
+ TosaArgGen.agNone,
),
"types": [DType.INT32],
"error_if_validators": (
@@ -3327,8 +3344,8 @@
"build_fcn": (
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
- TosaTensorValuesGen.tvgDefault,
- None,
+ TosaTensorValuesGen.tvgLazyGenDefault,
+ TosaArgGen.agNone,
),
"types": TYPE_BOOL,
"error_if_validators": (
@@ -3348,7 +3365,7 @@
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
TosaTensorValuesGen.tvgLogicalShift,
- None,
+ TosaArgGen.agNone,
),
"types": TYPE_INT,
"error_if_validators": (
@@ -3368,7 +3385,7 @@
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
TosaTensorValuesGen.tvgLogicalShift,
- None,
+ TosaArgGen.agNone,
),
"types": TYPE_INT,
"error_if_validators": (
@@ -3387,8 +3404,8 @@
"build_fcn": (
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
- TosaTensorValuesGen.tvgDefault,
- None,
+ TosaTensorValuesGen.tvgLazyGenDefault,
+ TosaArgGen.agNone,
),
"types": TYPE_BOOL,
"error_if_validators": (
@@ -3407,8 +3424,8 @@
"build_fcn": (
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
- TosaTensorValuesGen.tvgDefault,
- None,
+ TosaTensorValuesGen.tvgLazyGenDefault,
+ TosaArgGen.agNone,
),
"types": TYPE_BOOL,
"error_if_validators": (
@@ -3427,8 +3444,8 @@
"build_fcn": (
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
- TosaTensorValuesGen.tvgDefault,
- None,
+ TosaTensorValuesGen.tvgLazyGenDefault,
+ TosaArgGen.agNone,
),
"types": TYPE_FI32,
"error_if_validators": (
@@ -3440,6 +3457,9 @@
TosaErrorValidator.evDimensionMismatch,
TosaErrorValidator.evBroadcastShapesMismatch,
),
+ "data_gen": {
+ "fp": (gtu.DataGenType.PSEUDO_RANDOM,),
+ },
},
"minimum": {
"op": Op.MINIMUM,
@@ -3447,8 +3467,8 @@
"build_fcn": (
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
- TosaTensorValuesGen.tvgDefault,
- None,
+ TosaTensorValuesGen.tvgLazyGenDefault,
+ TosaArgGen.agNone,
),
"types": TYPE_FI32,
"error_if_validators": (
@@ -3460,6 +3480,9 @@
TosaErrorValidator.evDimensionMismatch,
TosaErrorValidator.evBroadcastShapesMismatch,
),
+ "data_gen": {
+ "fp": (gtu.DataGenType.PSEUDO_RANDOM,),
+ },
},
"mul": {
"op": Op.MUL,
@@ -3491,8 +3514,8 @@
"build_fcn": (
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
- TosaTensorValuesGen.tvgDefault,
- None,
+ TosaTensorValuesGen.tvgLazyGenDefault,
+ TosaArgGen.agNone,
),
"types": TYPE_FP,
"error_if_validators": (
@@ -3512,7 +3535,7 @@
build_binary_broadcast,
TosaTensorGen.tgBroadcastFuzz,
TosaTensorValuesGen.tvgAddSub,
- None,
+ TosaArgGen.agNone,
),
"types": TYPE_FI32,
"error_if_validators": (
@@ -3524,6 +3547,10 @@
TosaErrorValidator.evDimensionMismatch,
TosaErrorValidator.evBroadcastShapesMismatch,
),
+ "data_gen": {
+ "fp": (gtu.DataGenType.PSEUDO_RANDOM,),
+ },
+ "compliance": {"ulp": 0.5},
},
"table": {
"op": Op.TABLE,
@@ -4379,7 +4406,7 @@
build_rfft2d,
TosaTensorGen.tgRFFT2d,
TosaTensorValuesGen.tvgDefault,
- TosaArgGen.agNone,
+ None,
),
"types": [DType.FP32],
"error_if_validators": (
diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py
index d6598fb..d01e8a7 100644
--- a/verif/generator/tosa_verif_build_tests.py
+++ b/verif/generator/tosa_verif_build_tests.py
@@ -301,32 +301,42 @@
results = []
for test_type in testType:
testList = []
- for op in ttg.TOSA_OP_LIST:
- if re.match(args.filter + ".*", op):
- testList.extend(
- ttg.genOpTestList(
- op,
- shapeFilter=args.target_shapes,
- rankFilter=args.target_ranks,
- dtypeFilter=args.target_dtypes,
- testType=test_type,
+ try:
+ for opName in ttg.TOSA_OP_LIST:
+ if re.match(args.filter + ".*", opName):
+ testList.extend(
+ ttg.genOpTestList(
+ opName,
+ shapeFilter=args.target_shapes,
+ rankFilter=args.target_ranks,
+ dtypeFilter=args.target_dtypes,
+ testType=test_type,
+ )
)
- )
+ except Exception as e:
+ print(f"INTERNAL ERROR: Failure generating test lists for {opName}")
+ raise e
print("{} matching {} tests".format(len(testList), test_type))
testStrings = []
- for opName, testStr, dtype, error, shapeList, testArgs in testList:
- # Check for and skip duplicate tests
- if testStr in testStrings:
- print(f"Skipping duplicate test: {testStr}")
- continue
- else:
- testStrings.append(testStr)
+ try:
+ for opName, testStr, dtype, error, shapeList, testArgs in testList:
+ # Check for and skip duplicate tests
+ if testStr in testStrings:
+ print(f"Skipping duplicate test: {testStr}")
+ continue
+ else:
+ testStrings.append(testStr)
- results.append(
- ttg.serializeTest(opName, testStr, dtype, error, shapeList, testArgs)
- )
+ results.append(
+ ttg.serializeTest(
+ opName, testStr, dtype, error, shapeList, testArgs
+ )
+ )
+ except Exception as e:
+ print(f"INTERNAL ERROR: Failure creating test output for {opName}")
+ raise e
print(f"Done creating {len(results)} tests")