Minor fixes & add FP16 support to refmodel testing and conformance gen
Improve base inference conformance generation:
* Change to target specific dtypes required for conformance tests.
* Reduce dimension sizes of ERROR_IF tests.
NOTE: Will impact tensor, clamp and all ERROR_IF tests.
Add option to change seed on conformance generation for extra
testing.
Stop creation of convolution tests with negative output dimensions.
Improve reporting on failing to allocate tensor due to above issue.
Fix runner to correctly pass ref model debug flags.
Update reference_model examples.
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I992180dcfe265a7d50edfb151c9f38eeaef5c369
diff --git a/verif/conformance/test_select.py b/verif/conformance/test_select.py
index 8b60fbb..9e73b0d 100644
--- a/verif/conformance/test_select.py
+++ b/verif/conformance/test_select.py
@@ -336,7 +336,7 @@
"""Test selector for the AVG_POOL2D operator."""
name = "avg_pool2d"
- param_names = ["shape", "type", "stride", "kernel", "pad"]
+ param_names = ["shape", "type", "accum_type", "stride", "kernel", "pad"]
class BitwiseAndOperator(Operator):
@@ -380,7 +380,6 @@
"""Test selector for the CLZ operator."""
name = "clz"
- param_names = ["shape", "type"]
class ConcatOperator(Operator):
@@ -407,21 +406,21 @@
"""Test selector for the CONV2D operator."""
name = "conv2d"
- param_names = ["kernel", "shape", "type", "stride", "pad", "dilation"]
+ param_names = ["kernel", "shape", "type", "accum_type", "stride", "pad", "dilation"]
class Conv3dOperator(Operator):
"""Test selector for the CONV3D operator."""
name = "conv3d"
- param_names = ["kernel", "shape", "type", "stride", "pad", "dilation"]
+ param_names = ["kernel", "shape", "type", "accum_type", "stride", "pad", "dilation"]
class DepthwiseConv2dOperator(Operator):
"""Test selector for the DEPTHWISE_CONV2D operator."""
name = "depthwise_conv2d"
- param_names = ["kernel", "shape", "type", "stride", "pad", "dilation"]
+ param_names = ["kernel", "shape", "type", "accum_type", "stride", "pad", "dilation"]
class EqualOperator(Operator):
@@ -434,6 +433,7 @@
"""Test selector for the FULLY_CONNECTED operator."""
name = "fully_connected"
+ param_names = ["shape", "type", "accum_type"]
class GatherOperator(Operator):
@@ -511,6 +511,7 @@
"""Test selector for the MATMUL operator."""
name = "matmul"
+ param_names = ["shape", "type", "accum_type"]
class MaximumOperator(Operator):
@@ -684,7 +685,15 @@
"""Test selector for the TRANSPOSE_CONV2D operator."""
name = "transpose_conv2d"
- param_names = ["kernel", "shape", "type", "stride", "pad", "out_shape"]
+ param_names = [
+ "kernel",
+ "shape",
+ "type",
+ "accum_type",
+ "stride",
+ "pad",
+ "out_shape",
+ ]
def path_params(self, path):
"""Return a dictionary of params from the test path."""
diff --git a/verif/conformance/tosa_base_profile_ops_info.json b/verif/conformance/tosa_base_profile_ops_info.json
index 2af4c47..9dd5592 100644
--- a/verif/conformance/tosa_base_profile_ops_info.json
+++ b/verif/conformance/tosa_base_profile_ops_info.json
@@ -25,12 +25,12 @@
"5"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,2,65533,1",
"--target-shape",
- "1,65539,1,1,1",
- "--target-dtype",
- "int32"
+ "1,65539,1,1,1"
]
],
"params": {},
@@ -69,12 +69,12 @@
"5"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,2,65533,1",
"--target-shape",
- "1,65539,1,1,1",
- "--target-dtype",
- "int32"
+ "1,65539,1,1,1"
]
],
"params": {},
@@ -119,12 +119,12 @@
"2"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,3,1,65535",
"--target-shape",
"1,65538,1,2",
- "--target-dtype",
- "int8",
"--num-rand-permutations",
"2"
]
@@ -143,6 +143,12 @@
"group": "ew_binary",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
+ "--target-dtype",
+ "int32",
"--tensor-dim-range",
"16,64",
"--target-rank",
@@ -153,6 +159,12 @@
"3"
],
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
+ "--target-dtype",
+ "int32",
"--tensor-dim-range",
"1,16",
"--target-rank",
@@ -161,12 +173,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,2,65533,1",
"--target-shape",
- "1,65539,1,1,1",
- "--target-dtype",
- "int8"
+ "1,65539,1,1,1"
]
],
"params": {},
@@ -184,6 +196,10 @@
"group": "tensor",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
"--target-shape",
"1,49,42,28",
"--target-shape",
@@ -191,12 +207,12 @@
"--allow-pooling-and-conv-oversizes"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,65535,5,1",
"--target-shape",
"1,3,65537,1",
- "--target-dtype",
- "int8",
"--allow-pooling-and-conv-oversizes"
]
],
@@ -224,6 +240,12 @@
"group": "ew_binary",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
+ "--target-dtype",
+ "int32",
"--tensor-dim-range",
"1,64",
"--target-rank",
@@ -244,12 +266,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,2,65536,1",
"--target-shape",
- "1,65539,1,1,1",
- "--target-dtype",
- "int8"
+ "1,65539,1,1,1"
]
],
"params": {},
@@ -266,6 +288,12 @@
"group": "ew_unary",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
+ "--target-dtype",
+ "int32",
"--tensor-dim-range",
"2,64",
"--target-rank",
@@ -286,12 +314,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,2,65537,1",
"--target-shape",
- "1,65538,1,1,1",
- "--target-dtype",
- "int8"
+ "1,65538,1,1,1"
]
],
"params": {},
@@ -308,6 +336,12 @@
"group": "ew_binary",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
+ "--target-dtype",
+ "int32",
"--tensor-dim-range",
"3,64",
"--target-rank",
@@ -328,12 +362,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,2,65538,1",
"--target-shape",
- "1,65537,1,1,1",
- "--target-dtype",
- "int8"
+ "1,65537,1,1,1"
]
],
"params": {},
@@ -350,6 +384,12 @@
"group": "ew_binary",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
+ "--target-dtype",
+ "int32",
"--tensor-dim-range",
"4,64",
"--target-rank",
@@ -370,12 +410,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,2,65536,1",
"--target-shape",
- "1,65536,1,1,1",
- "--target-dtype",
- "int8"
+ "1,65536,1,1,1"
]
],
"params": {},
@@ -426,12 +466,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65535,1,1",
"--target-shape",
- "1,65538,1,2",
- "--target-dtype",
- "int8"
+ "1,65538,1,2"
]
],
"params": {
@@ -458,6 +498,10 @@
"group": "activation",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
"--tensor-dim-range",
"20,64",
"--target-rank",
@@ -468,6 +512,10 @@
"3"
],
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
"--tensor-dim-range",
"1,20",
"--target-rank",
@@ -476,12 +524,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,2,1,65536",
"--target-shape",
- "1,3,65537,1,1",
- "--target-dtype",
- "int8"
+ "1,3,65537,1,1"
]
],
"params": {},
@@ -520,12 +568,12 @@
"5"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,2,65533,1",
"--target-shape",
- "1,65539,1,1,1",
- "--target-dtype",
- "int32"
+ "1,65539,1,1,1"
]
],
"params": {},
@@ -580,12 +628,12 @@
"1"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65535,1,1",
"--target-shape",
"1,65538,1,2",
- "--target-dtype",
- "int8",
"--num-const-inputs-concat",
"2"
]
@@ -614,10 +662,10 @@
"int32"
],
[
- "--target-shape",
- "3,22,17,1,2",
"--target-dtype",
- "int32"
+ "int32",
+ "--target-shape",
+ "3,22,17,1,2"
]
],
"params": {},
@@ -668,12 +716,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65537,1,1",
"--target-shape",
- "1,65530,1,2",
- "--target-dtype",
- "int8"
+ "1,65530,1,2"
]
],
"params": {},
@@ -690,6 +738,10 @@
"group": "tensor",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
"--target-shape",
"1,49,42,28",
"--target-shape",
@@ -697,12 +749,12 @@
"--allow-pooling-and-conv-oversizes"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,65535,4,1",
"--target-shape",
"1,5,65536,1",
- "--target-dtype",
- "int8",
"--max-conv-dilation",
"1",
"--allow-pooling-and-conv-oversizes"
@@ -735,23 +787,23 @@
"group": "tensor",
"generator_args": [
[
- "--target-shape",
- "1,9,21,14,1",
- "--target-shape",
- "1,9,11,12,3",
"--target-dtype",
"int8",
"--target-dtype",
"int16",
+ "--target-shape",
+ "1,9,21,14,1",
+ "--target-shape",
+ "1,9,11,12,3",
"--allow-pooling-and-conv-oversizes"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65535,3,1",
"--target-shape",
"1,1,3,65536,1",
- "--target-dtype",
- "int8",
"--max-conv-dilation",
"1",
"--allow-pooling-and-conv-oversizes"
@@ -785,6 +837,10 @@
"group": "tensor",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
"--target-shape",
"1,9,44,3",
"--target-shape",
@@ -792,12 +848,12 @@
"--allow-pooling-and-conv-oversizes"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,65532,2,1",
"--target-shape",
"1,4,65537,1",
- "--target-dtype",
- "int8",
"--max-conv-dilation",
"1",
"--allow-pooling-and-conv-oversizes"
@@ -852,12 +908,12 @@
"5"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,2,65501,1",
"--target-shape",
- "1,65541,1,1,1",
- "--target-dtype",
- "int32"
+ "1,65541,1,1,1"
]
],
"params": {},
@@ -880,24 +936,24 @@
"int16"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"444,1",
"--target-shape",
"3,65538",
"--tensor-dim-range",
- "1,2",
- "--target-dtype",
- "int8"
+ "1,2"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"3,16",
"--target-shape",
"1,23",
"--tensor-dim-range",
- "100,200",
- "--target-dtype",
- "int8"
+ "100,200"
]
],
"params": {},
@@ -922,14 +978,14 @@
"4,64"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"2,65533,1",
"--target-shape",
"1,2,65533",
"--target-shape",
- "5000,1,1",
- "--target-dtype",
- "int8"
+ "5000,1,1"
]
],
"params": {},
@@ -974,12 +1030,12 @@
[
"--filter",
"^greater$",
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,2,65530,1",
"--target-shape",
- "1,65539,1,2,1",
- "--target-dtype",
- "int32"
+ "1,65539,1,2,1"
]
],
"params": {},
@@ -1018,12 +1074,12 @@
"5"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,2,65533,1",
"--target-shape",
- "1,65539,1,3,1",
- "--target-dtype",
- "int32"
+ "1,65539,1,3,1"
]
],
"params": {},
@@ -1075,12 +1131,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65536,1,1",
"--target-shape",
- "1,65531,1,2",
- "--target-dtype",
- "int8"
+ "1,65531,1,2"
]
],
"params": {},
@@ -1119,12 +1175,12 @@
"5"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,65533,1",
"--target-shape",
- "1,1,65539,1,1",
- "--target-dtype",
- "int32"
+ "1,1,65539,1,1"
]
],
"params": {},
@@ -1141,6 +1197,8 @@
"group": "ew_binary",
"generator_args": [
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"1,64",
"--target-rank",
@@ -1151,6 +1209,8 @@
"3"
],
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"1,16",
"--target-rank",
@@ -1159,6 +1219,8 @@
"5"
],
[
+ "--target-dtype",
+ "bool",
"--target-shape",
"1,65538,1",
"--target-shape",
@@ -1209,12 +1271,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,65532,1,1",
"--target-shape",
- "1,2,1,65538",
- "--target-dtype",
- "int8"
+ "1,2,1,65538"
]
],
"params": {},
@@ -1231,6 +1293,8 @@
"group": "ew_unary",
"generator_args": [
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"2,64",
"--target-rank",
@@ -1241,6 +1305,8 @@
"3"
],
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"1,15",
"--target-rank",
@@ -1249,6 +1315,8 @@
"5"
],
[
+ "--target-dtype",
+ "bool",
"--target-shape",
"1,65537,1",
"--target-shape",
@@ -1269,6 +1337,8 @@
"group": "ew_binary",
"generator_args": [
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"3,64",
"--target-rank",
@@ -1279,6 +1349,8 @@
"3"
],
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"1,14",
"--target-rank",
@@ -1287,6 +1359,8 @@
"5"
],
[
+ "--target-dtype",
+ "bool",
"--target-shape",
"1,65536,1",
"--target-shape",
@@ -1337,12 +1411,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65540,3",
"--target-shape",
- "2,1,65530,1,1",
- "--target-dtype",
- "int8"
+ "2,1,65530,1,1"
]
],
"params": {},
@@ -1359,6 +1433,8 @@
"group": "ew_binary",
"generator_args": [
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"4,64",
"--target-rank",
@@ -1369,6 +1445,8 @@
"3"
],
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"3,16",
"--target-rank",
@@ -1377,6 +1455,8 @@
"5"
],
[
+ "--target-dtype",
+ "bool",
"--target-shape",
"1,65539,1",
"--target-shape",
@@ -1413,14 +1493,14 @@
"1,16"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,65532,1",
"--target-shape",
"1,2,65538",
"--target-shape",
- "2000,1,1",
- "--target-dtype",
- "int8"
+ "2000,1,1"
]
],
"params": {},
@@ -1459,12 +1539,12 @@
"5"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,2,65533,1",
"--target-shape",
- "1,65539,1,1,1",
- "--target-dtype",
- "int32"
+ "1,65539,1,1,1"
]
],
"params": {},
@@ -1481,6 +1561,10 @@
"group": "tensor",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
"--target-shape",
"1,4,75,3",
"--target-shape",
@@ -1488,14 +1572,14 @@
"--allow-pooling-and-conv-oversizes"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,65535,5,1",
"--target-shape",
"1,3,65537,1",
"--target-shape",
"33333,3,2,1",
- "--target-dtype",
- "int8",
"--allow-pooling-and-conv-oversizes"
]
],
@@ -1545,12 +1629,12 @@
"5"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,2,65533,1",
"--target-shape",
- "1,65539,1,1,1",
- "--target-dtype",
- "int32"
+ "1,65539,1,1,1"
]
],
"params": {},
@@ -1597,12 +1681,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,2,65500,1",
"--target-shape",
- "1,68539,1,1,1",
- "--target-dtype",
- "int8"
+ "1,68539,1,1,1"
]
],
"params": {},
@@ -1651,12 +1735,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"3,1,65540,1",
"--target-shape",
- "1,1,2,1,65531",
- "--target-dtype",
- "int8"
+ "1,1,2,1,65531"
]
],
"params": {},
@@ -1673,8 +1757,6 @@
"group": "data_layout",
"generator_args": [
[
- "--tensor-dim-range",
- "1,64",
"--target-dtype",
"int8",
"--target-dtype",
@@ -1683,6 +1765,8 @@
"int32",
"--target-dtype",
"bool",
+ "--tensor-dim-range",
+ "1,64",
"--target-rank",
"1",
"--target-rank",
@@ -1691,8 +1775,6 @@
"3"
],
[
- "--tensor-dim-range",
- "1,16",
"--target-dtype",
"int8",
"--target-dtype",
@@ -1701,22 +1783,24 @@
"int32",
"--target-dtype",
"bool",
+ "--tensor-dim-range",
+ "1,16",
"--target-rank",
"4"
],
[
+ "--target-dtype",
+ "int16",
"--tensor-dim-range",
"1,16",
"--target-rank",
- "5",
- "--target-dtype",
- "int16"
+ "5"
],
[
- "--target-shape",
- "1,1,65537,1",
"--target-dtype",
- "int8"
+ "int8",
+ "--target-shape",
+ "1,1,65537,1"
]
],
"params": {},
@@ -1743,10 +1827,14 @@
"group": "reduction",
"generator_args": [
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"1,32"
],
[
+ "--target-dtype",
+ "bool",
"--target-shape",
"65530,1,1,1",
"--target-shape",
@@ -1768,10 +1856,14 @@
"group": "reduction",
"generator_args": [
[
+ "--target-dtype",
+ "bool",
"--tensor-dim-range",
"1,32"
],
[
+ "--target-dtype",
+ "bool",
"--target-shape",
"65530,1,1,1",
"--target-shape",
@@ -1803,12 +1895,12 @@
"1,32"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"65530,1,1,1",
"--target-shape",
- "2,65538,1,1",
- "--target-dtype",
- "int8"
+ "2,65538,1,1"
]
],
"params": {},
@@ -1836,12 +1928,12 @@
"1,24"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65538,1",
"--target-shape",
- "2,65535,1,1",
- "--target-dtype",
- "int8"
+ "2,65535,1,1"
]
],
"params": {},
@@ -1865,12 +1957,12 @@
"1,32"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"3,65532,1,1",
"--target-shape",
- "65536,1,1,2",
- "--target-dtype",
- "int32"
+ "65536,1,1,2"
]
],
"params": {},
@@ -1888,6 +1980,18 @@
"group": "type_conversion",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
+ "--target-dtype",
+ "int32",
+ "--target-dtype",
+ "int48",
+ "--target-dtype",
+ "uint8",
+ "--target-dtype",
+ "uint16",
"--tensor-dim-range",
"16,32",
"--target-rank",
@@ -1898,6 +2002,18 @@
"3"
],
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
+ "--target-dtype",
+ "int32",
+ "--target-dtype",
+ "int48",
+ "--target-dtype",
+ "uint8",
+ "--target-dtype",
+ "uint16",
"--tensor-dim-range",
"1,16",
"--target-rank",
@@ -1906,12 +2022,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,2,65533,1",
"--target-shape",
- "1,65539,1,1",
- "--target-dtype",
- "int8"
+ "1,65539,1,1"
]
],
"params": {
@@ -1967,12 +2083,12 @@
"2"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65535,1",
"--target-shape",
"1,65538,1,2",
- "--target-dtype",
- "int8",
"--num-rand-permutations",
"2"
]
@@ -1999,10 +2115,10 @@
"int16"
],
[
- "--target-shape",
- "1,3,16383,1",
"--target-dtype",
"int8",
+ "--target-shape",
+ "1,3,16383,1",
"--max-resize-output-dim",
"9500"
]
@@ -2033,12 +2149,12 @@
"1,32"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65535",
"--target-shape",
- "1,65540,1,2",
- "--target-dtype",
- "int8"
+ "1,65540,1,2"
]
],
"params": {},
@@ -2063,14 +2179,14 @@
"4,64"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,65540,1",
"--target-shape",
"1,1,65533",
"--target-shape",
- "3000,7,1",
- "--target-dtype",
- "int8"
+ "3000,7,1"
]
],
"params": {},
@@ -2121,12 +2237,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65535,1",
"--target-shape",
"1,65538,1,2",
- "--target-dtype",
- "int8",
"--num-rand-permutations",
"2"
]
@@ -2181,12 +2297,12 @@
"2"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,1,65535,1",
"--target-shape",
"1,65538,1,2",
- "--target-dtype",
- "int8",
"--num-rand-permutations",
"2"
]
@@ -2228,12 +2344,12 @@
"5"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"1,2,65533,1",
"--target-shape",
- "1,65539,1,1,1",
- "--target-dtype",
- "int32"
+ "1,65539,1,1,1"
]
],
"params": {},
@@ -2250,6 +2366,10 @@
"group": "ew_binary",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
"--tensor-dim-range",
"16,64",
"--target-rank",
@@ -2260,6 +2380,10 @@
"3"
],
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
"--tensor-dim-range",
"1,16",
"--target-rank",
@@ -2268,12 +2392,12 @@
"5"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,2,65536,1",
"--target-shape",
- "1,65537,1,1,3",
- "--target-dtype",
- "int8"
+ "1,65537,1,1,3"
]
],
"params": {},
@@ -2298,12 +2422,12 @@
"4,32"
],
[
+ "--target-dtype",
+ "int8",
"--target-shape",
"65533",
"--target-shape",
- "1,2,65540,1",
- "--target-dtype",
- "int8"
+ "1,2,65540,1"
]
],
"params": {},
@@ -2352,12 +2476,12 @@
[
"--filter",
"^transpose$",
+ "--target-dtype",
+ "int8",
"--target-shape",
"1,65537,1,2",
"--target-shape",
"65535,1,1,1",
- "--target-dtype",
- "int8",
"--num-rand-permutations",
"1"
]
@@ -2377,21 +2501,21 @@
"group": "tensor",
"generator_args": [
[
+ "--target-dtype",
+ "int8",
+ "--target-dtype",
+ "int16",
"--target-shape",
"1,49,33,1",
"--target-shape",
"2,11,33,3",
- "--target-dtype",
- "int16",
- "--target-dtype",
- "int8",
"--allow-pooling-and-conv-oversizes"
],
[
- "--target-shape",
- "1,65536,1,1",
"--target-dtype",
"int8",
+ "--target-shape",
+ "1,65536,1,1",
"--max-conv-dilation",
"1"
]
@@ -2427,12 +2551,12 @@
"int32"
],
[
+ "--target-dtype",
+ "int32",
"--target-shape",
"3,22,17,1,2",
"--target-shape",
- "1,65537,2,1",
- "--target-dtype",
- "int32"
+ "1,65537,2,1"
]
],
"params": {},
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index c057b73..7032ad4 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -34,12 +34,14 @@
"tosa-bi": {
"operator_test_params": "tosa_base_profile_ops_info.json",
"framework_tests": "tosa_base_profile_framework_ops_info.json",
- "exclude_types": ["float"],
+ "exclude_types": [],
}
}
LOCATION_REF_MODEL_BINARY = Path("build/reference_model/tosa_reference_model")
+DEFAULT_SEED = 42
+
class GenConformanceError(Exception):
"""Generation error reporting exception."""
@@ -90,7 +92,7 @@
"-o",
str(op_build_dir),
"--seed",
- "42",
+ str(args.random_seed),
]
ref_cmds = []
@@ -99,12 +101,26 @@
# Append extra parameters and run test generator for each set of parameters.
for arglist in test_params[operator]["generator_args"]:
ref_cmd_pos_test = ref_cmd_base.copy()
+ ref_cmd_pos_test.extend(["--test-type", "positive"])
ref_cmd_pos_test.extend(arglist)
ref_cmds.append(ref_cmd_pos_test)
if args.test_type in ["negative", "both"]:
+ # Get target-dtypes options only to limit tests to those needed
+ target_dtypes_args = []
+ for arglist in test_params[operator]["generator_args"]:
+ idx = 0
+ while idx < len(arglist):
+ if arglist[idx] == "--target-dtype":
+ if arglist[idx + 1] not in target_dtypes_args:
+ target_dtypes_args.extend(arglist[idx : idx + 2])
+ idx += 1 # skip over option (and then argument below)
+ idx += 1
ref_cmd_neg_test = ref_cmd_base.copy()
ref_cmd_neg_test.extend(["--test-type", "negative"])
+ # Limit sizes of negative tests
+ ref_cmd_neg_test.extend(["--tensor-dim-range", "1,16"])
+ ref_cmd_neg_test.extend(target_dtypes_args)
ref_cmds.append(ref_cmd_neg_test)
logger.debug(f"Creating {operator} tests with {len(ref_cmds)} parameter(s)")
@@ -430,6 +446,13 @@
help="Reference Model directory (must be pre-built)",
)
parser.add_argument(
+ "--seed",
+ dest="random_seed",
+ default=DEFAULT_SEED,
+ type=int,
+ help="Random test seed",
+ )
+ parser.add_argument(
"--framework-tests-directory",
dest="framework_tests_dir",
type=Path,
@@ -535,6 +558,11 @@
print(f"Creating conformance tests for TOSA {args.profile} profile")
print(f"Output directory: {args.output_dir}")
+ if args.random_seed != DEFAULT_SEED:
+ logger.warning(
+ "Random test seed changed from default, tests will not match official conformance"
+ )
+
args.build_dir = args.build_dir.resolve()
logger.debug(f"Creating build directory: {args.build_dir}")
args.build_dir.mkdir(parents=True, exist_ok=True)
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index 69968d3..e0c6cf0 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -1017,7 +1017,7 @@
s_vals = [testGen.rng.choice(range(-5, 0))]
else:
# Stride must be greater than 1 to force non-integer error
- startStride = 1 if error_name != ErrorIf.PoolingOutputShapeNonInteger else 2
+ startStride = 1 if error_name != ErrorIf.ConvOutputShapeNonInteger else 2
s_vals = [x for x in range(startStride, testGen.args.max_conv_stride + 1)]
strides = {x for x in itertools.product(*([s_vals] * k_rank))}
if error_name == ErrorIf.DilationSmallerOne:
@@ -1058,18 +1058,14 @@
for d in sorted(list(dilations)):
if (
n % sparsity == 0
- # padding must not exceed the kernel size ?
- # and p[0] < k[0] and p[1] < k[0]
- # and p[2] < k[1] and p[3] < k[1]
- # and (k_rank < 3 or (p[4] < k[2] and p[5] < k[2]))
- # the padded shape must exceed the kernel size
- and (ifm_shape[1] + p[0] + p[1]) > k[0]
- and (ifm_shape[2] + p[2] + p[3]) > k[1]
- and (k_rank < 3 or ((ifm_shape[3] + p[4] + p[5]) > k[2]))
- # the padded shape must exceed the dilation
- and (ifm_shape[1] + p[0] + p[1]) > d[0]
- and (ifm_shape[2] + p[2] + p[3]) > d[1]
- and (k_rank < 3 or ((ifm_shape[3] + p[4] + p[5]) > d[2]))
+ # the padded shape must exceed the dilation * kernel to get a positive
+ # sized output shape
+ and (ifm_shape[1] - 1 + p[0] + p[1]) > d[0] * (k[0] - 1)
+ and (ifm_shape[2] - 1 + p[2] + p[3]) > d[1] * (k[1] - 1)
+ and (
+ k_rank < 3
+ or ((ifm_shape[3] - 1 + p[4] + p[5]) > d[2] * (k[2] - 1))
+ )
):
remainders = []
for index in range(k_rank):
diff --git a/verif/runner/tosa_refmodel_sut_run.py b/verif/runner/tosa_refmodel_sut_run.py
index 2ae7a69..6acaaf4 100644
--- a/verif/runner/tosa_refmodel_sut_run.py
+++ b/verif/runner/tosa_refmodel_sut_run.py
@@ -40,10 +40,10 @@
# Specific debug options for reference model
if args.ref_debug:
- cmd.extend(["-d ALL", "-l {}".format(args.ref_debug)])
+ cmd.extend(["-d", "ALL", "-l", args.ref_debug])
if args.ref_intermediates:
- cmd.extend(["-D dump_intermediates=1"])
+ cmd.extend(["--dump_intermediates", str(args.ref_intermediates)])
# Run command and interpret tosa graph result via process return codes
graphMessage = None
diff --git a/verif/tests/test_tosa_refmodel.py b/verif/tests/test_tosa_refmodel.py
index b2f33dd..fbe3a7d 100644
--- a/verif/tests/test_tosa_refmodel.py
+++ b/verif/tests/test_tosa_refmodel.py
@@ -46,6 +46,7 @@
"int16": "i16",
"int32": "i32",
"float": "float",
+ "fp16": "f16",
}
@@ -125,16 +126,19 @@
("add", "float", 1),
("abs", "int32", 1),
("abs", "float", 1),
+ ("abs", "fp16", 1),
("negate", "int8", 1),
("negate", "int16", 1),
("negate", "int32", 1),
("negate", "float", 1),
+ ("negate", "fp16", 1),
# One test per axis (shape dimensions)
("concat", "bool", SHAPE_DIMS),
("concat", "int8", SHAPE_DIMS),
("concat", "int16", SHAPE_DIMS),
("concat", "int32", SHAPE_DIMS),
("concat", "float", SHAPE_DIMS),
+ ("concat", "fp16", SHAPE_DIMS),
]