Update verifier library data-type support
Make compliance meta-data data-type required for all.
Add data-type checking for all verifier modes.
Add initial enum support for new ROUND compliance mode.
Improve print out information from library.
Use numpy ctypes.data_as to get f16 support compared to ctypes_lib.
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Ie983ba4ea958a88556f30c09b3ebc19cd9ec96b7
diff --git a/reference_model/src/verify/verify_dot_product.cc b/reference_model/src/verify/verify_dot_product.cc
index a96befa..39895b1 100644
--- a/reference_model/src/verify/verify_dot_product.cc
+++ b/reference_model/src/verify/verify_dot_product.cc
@@ -105,9 +105,6 @@
TOSA_REF_REQUIRE(refBnd != nullptr, "reference bounds tensor is missing");
TOSA_REF_REQUIRE(imp != nullptr, "implementation tensor is missing");
- // Validate data-type
- TOSA_REF_REQUIRE(dpInfo.dataType == mapToDType(imp->data_type), "invalid data type in config");
-
// Get number of dot-product elements
const int64_t T = numElements(std::vector<int32_t>(ref->shape, ref->shape + ref->num_dims));
TOSA_REF_REQUIRE(T > 0, "invalid shape for reference tensor");
@@ -124,8 +121,10 @@
return validateData(refData, refBndData, impData, static_cast<size_t>(T), dpInfo);
break;
}
- default:
+ default: {
+ WARNING("tosa verifier: data-type not supported.");
break;
+ }
}
return false;
diff --git a/reference_model/src/verify/verify_entry.cc b/reference_model/src/verify/verify_entry.cc
index 1f7c680..32614b6 100644
--- a/reference_model/src/verify/verify_entry.cc
+++ b/reference_model/src/verify/verify_entry.cc
@@ -38,7 +38,7 @@
return verifyULP(ref, imp, cfg.ulpInfo.ulp);
}
default: {
- WARNING("unsupported verification mode.");
+ WARNING("tosa verifier: unsupported verification mode.");
break;
}
}
@@ -57,40 +57,58 @@
// Check inputs for nullptr
if (!ref || !imp || !config_json)
{
- WARNING("one of the inputs is missing.");
+ WARNING("tosa verifier: one of the inputs is missing.");
return false;
}
// Extract verification config
if (!ref->name)
{
- WARNING("tensor name is not specified.");
+ WARNING("tosa verifier: tensor name is not specified.");
return false;
}
auto cfg = TosaReference::parseVerifyConfig(ref->name, config_json);
if (!cfg)
{
- WARNING("invalid json config.");
+ WARNING("tosa verifier: invalid json config.");
return false;
}
// Validate shape
if (ref->num_dims != imp->num_dims)
{
- WARNING("tensors have different number of dimensions.");
+ WARNING("tosa verifier: tensors have different number of dimensions.");
return false;
}
if (!ref->shape || !imp->shape)
{
- WARNING("one of tensors' shape is missing.");
+ WARNING("tosa verifier: one of tensors' shape is missing.");
return false;
}
if (std::vector(ref->shape, ref->shape + ref->num_dims) != std::vector(imp->shape, imp->shape + imp->num_dims))
{
- WARNING("tensors have different shapes.");
+ WARNING("tosa verifier: tensors have different shapes.");
return false;
}
+ // Validate data-type
+ if (ref->data_type == tosa_datatype_fp64_t)
+ {
+ if (cfg->dataType != TosaReference::mapToDType(imp->data_type))
+ {
+ WARNING("tosa verifier: incorrect tensor data type.");
+ return false;
+ }
+ }
+ else
+ {
+ if (ref->data_type != imp->data_type)
+ {
+ WARNING("tosa verifier: tensors have different data types.");
+ return false;
+ }
+ }
+
// Run verification
return verify(ref, ref_bnd, imp, *cfg);
}
diff --git a/reference_model/src/verify/verify_exact.cc b/reference_model/src/verify/verify_exact.cc
index 7fde5bb..1a0c44c 100644
--- a/reference_model/src/verify/verify_exact.cc
+++ b/reference_model/src/verify/verify_exact.cc
@@ -44,7 +44,7 @@
});
}
default:
- WARNING("data-type not supported.");
+ WARNING("tosa verifier: data-type not supported.");
break;
}
diff --git a/reference_model/src/verify/verify_ulp.cc b/reference_model/src/verify/verify_ulp.cc
index 622fba4..223bc48 100644
--- a/reference_model/src/verify/verify_ulp.cc
+++ b/reference_model/src/verify/verify_ulp.cc
@@ -145,6 +145,7 @@
});
}
default:
+ WARNING("tosa verifier: data-type not supported.");
break;
}
diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc
index bb4feaa..786ab40 100644
--- a/reference_model/src/verify/verify_utils.cc
+++ b/reference_model/src/verify/verify_utils.cc
@@ -48,6 +48,7 @@
{ VerifyMode::DotProduct, "DOT_PRODUCT" },
{ VerifyMode::ReduceProduct, "REDUCE_PRODUCT" },
{ VerifyMode::FpSpecial, "FP_SPECIAL" },
+ { VerifyMode::Round, "ROUND" },
})
void from_json(const nlohmann::json& j, UlpInfo& ulpInfo)
@@ -57,7 +58,6 @@
void from_json(const nlohmann::json& j, DotProductVerifyInfo& dotProductInfo)
{
- j.at("data_type").get_to(dotProductInfo.dataType);
j.at("s").get_to(dotProductInfo.s);
j.at("ks").get_to(dotProductInfo.ks);
}
@@ -65,6 +65,7 @@
void from_json(const nlohmann::json& j, VerifyConfig& cfg)
{
j.at("mode").get_to(cfg.mode);
+ j.at("data_type").get_to(cfg.dataType);
if (j.contains("ulp_info"))
{
j.at("ulp_info").get_to(cfg.ulpInfo);
diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h
index 510b9cb..5b98f5c 100644
--- a/reference_model/src/verify/verify_utils.h
+++ b/reference_model/src/verify/verify_utils.h
@@ -26,7 +26,7 @@
#define TOSA_REF_REQUIRE(COND, MESSAGE) \
if (!(COND)) \
{ \
- WARNING(MESSAGE); \
+ WARNING("tosa verifier: " MESSAGE "."); \
return false; \
}
@@ -43,7 +43,8 @@
Ulp,
DotProduct,
ReduceProduct,
- FpSpecial
+ FpSpecial,
+ Round
};
/// \brief ULP verification meta-data
@@ -59,7 +60,6 @@
{
DotProductVerifyInfo() = default;
- DType dataType;
int32_t s;
int32_t ks;
};
@@ -70,6 +70,7 @@
VerifyConfig() = default;
VerifyMode mode;
+ DType dataType;
UlpInfo ulpInfo;
DotProductVerifyInfo dotProductInfo;
};
diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp
index 7482847..7b6ba9d 100644
--- a/reference_model/test/verify_tests.cpp
+++ b/reference_model/test/verify_tests.cpp
@@ -135,8 +135,8 @@
"tensors" : {
"out1" : {
"mode": "DOT_PRODUCT",
+ "data_type": "FP32",
"dot_product_info" : {
- "data_type": "FP32",
"s": 2,
"ks": 9
}
@@ -199,7 +199,8 @@
std::string json_cfg = R"({
"tensors" : {
"out1" : {
- "mode": "EXACT"
+ "mode": "EXACT",
+ "data_type": "FP32"
}
}
})";
@@ -243,6 +244,7 @@
"tensors" : {
"out1" : {
"mode": "ULP",
+ "data_type": "FP32",
"ulp_info": {
"ulp": 5
}
diff --git a/scripts/schemavalidation/compliance-config.schema.json b/scripts/schemavalidation/compliance-config.schema.json
index 677a2a8..570c88f 100644
--- a/scripts/schemavalidation/compliance-config.schema.json
+++ b/scripts/schemavalidation/compliance-config.schema.json
@@ -25,14 +25,19 @@
"description": "verifier mode EXACT, DOT_PRODUCT, ULP, REDUCE_PRODUCT, or FP_SPECIAL",
"type": "string"
},
+ "data_type": {
+ "description": "tensor data type, such as: BOOL, INT16, FP32",
+ "type": "string"
+ },
"ulp_info": {
"description": "info required for the ULP mode",
"type": "object",
"properties":
{
"ulp": {
- "description": "ulp range limit - such as 1 or 0.5",
- "type": "string"
+ "description": "ulp range limit - positive number",
+ "type": "integer",
+ "minimum": 0
}
},
"required": [ "ulp" ]
@@ -49,23 +54,19 @@
"ks": {
"description": "kernel size for this dot product operation",
"type": "integer"
- },
- "data_type": {
- "description": "tensor data type, such as: BOOL, INT16, FP32",
- "type": "string"
}
},
"additionalProperties": false,
"required": [
"s",
- "ks",
- "data_type"
+ "ks"
]
}
},
"additionalProperties": false,
"required": [
- "mode"
+ "mode",
+ "data_type"
]
}
}
diff --git a/verif/checker/verifier.py b/verif/checker/verifier.py
index 06ffcfb..684bea3 100644
--- a/verif/checker/verifier.py
+++ b/verif/checker/verifier.py
@@ -72,7 +72,7 @@
ct.cast(shape, ct.POINTER(ct.c_int32)),
ct.c_int32(len(array.shape)),
ct.c_int(NUMPY_DATATYPE_TO_CLIENTTYPE[array.dtype]["type"]),
- ct.cast(np.ctypeslib.as_ctypes(array), ct.POINTER(ct.c_uint8)),
+ array.ctypes.data_as(ct.POINTER(ct.c_uint8)),
ct.c_size_t(size_in_bytes),
)
return tensor
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index d15f785..8beb2ae 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -282,17 +282,21 @@
)
def tensorComplianceMetaData(self, op, argsDict, outputTensor, errorName):
- if errorName:
- # No compliance for error tests
+ if errorName or not gtu.dtypeIsFloat(outputTensor.dtype):
+ # No compliance for error tests or integer tests currently
return None
+
# Create compliance meta data for expected output tensor
- compliance_tens = {"mode": None}
+ compliance_tens = {
+ "mode": None,
+ # Data type is needed for all FP runs, as refmodel precise mode produces FP64
+ "data_type": gtu.DTYPE_ATTRIBUTES[outputTensor.dtype]["json"],
+ }
if argsDict["dg_type"] == gtu.DataGenType.DOT_PRODUCT:
mode = gtu.ComplianceMode.DOT_PRODUCT
compliance_tens["dot_product_info"] = {
"s": argsDict["s"],
"ks": argsDict["ks"],
- "data_type": gtu.DTYPE_ATTRIBUTES[outputTensor.dtype]["json"],
}
elif argsDict["dg_type"] == gtu.DataGenType.OP_SPECIAL:
mode = gtu.ComplianceMode.FP_SPECIAL
@@ -301,6 +305,8 @@
compliance_tens["ulp_info"] = {"ulp": op["compliance"]["ulp"]}
elif op["op"] == Op.REDUCE_PRODUCT:
mode = gtu.ComplianceMode.REDUCE_PRODUCT
+ elif op["op"] in (Op.ADD, Op.MUL, Op.SUB, Op.CEIL, Op.FLOOR, Op.CAST):
+ mode = gtu.ComplianceMode.ROUND
else:
mode = gtu.ComplianceMode.EXACT
compliance_tens["mode"] = gtu.ComplianceMode(mode).name
diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py
index 75a0df5..dddc320 100644
--- a/verif/generator/tosa_utils.py
+++ b/verif/generator/tosa_utils.py
@@ -38,6 +38,7 @@
ULP = 2
FP_SPECIAL = 3
REDUCE_PRODUCT = 4
+ ROUND = 5
class DataGenType(IntEnum):
diff --git a/verif/tests/test_schemavalidation.py b/verif/tests/test_schemavalidation.py
index 1ecd3ee..664e3a4 100644
--- a/verif/tests/test_schemavalidation.py
+++ b/verif/tests/test_schemavalidation.py
@@ -72,11 +72,7 @@
def test_schemavalidation_compliance_minimal():
json = {
"version": "v",
- "tensors": {
- "output": {
- "mode": "mode",
- }
- },
+ "tensors": {"output": {"mode": "mode", "data_type": "type"}},
}
sv = sch.TestDescSchemaValidator()
diff --git a/verif/tests/test_tosa_verifier.py b/verif/tests/test_tosa_verifier.py
index 996939c..864fa9c 100644
--- a/verif/tests/test_tosa_verifier.py
+++ b/verif/tests/test_tosa_verifier.py
@@ -40,7 +40,8 @@
"tensors": {
"output1": {
"mode": "DOT_PRODUCT",
- "dot_product_info": {"ks": 1000, "s": 0, "data_type": "FP32"},
+ "data_type": "FP32",
+ "dot_product_info": {"ks": 1000, "s": 0},
}
},
}