Main Compliance: Add RESIZE support
Add RELATIVE verify mode for RESIZE.
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I4fe352579507211dae7a048bf080c24426ce42a2
diff --git a/reference_model/CMakeLists.txt b/reference_model/CMakeLists.txt
index d2dce3c..0f806fc 100644
--- a/reference_model/CMakeLists.txt
+++ b/reference_model/CMakeLists.txt
@@ -81,6 +81,7 @@
src/verify/verify_entry.cc
src/verify/verify_exact.cc
src/verify/verify_reduce_product.cc
+ src/verify/verify_relative.cc
src/verify/verify_ulp.cc
src/verify/verify_utils.cc
src/ops/op_factory.cc
@@ -157,6 +158,7 @@
src/verify/verify_entry.cc
src/verify/verify_exact.cc
src/verify/verify_reduce_product.cc
+ src/verify/verify_relative.cc
src/verify/verify_ulp.cc
src/verify/verify_utils.cc
src/verify/verify_config.cc
diff --git a/reference_model/src/generate/generate_utils.cc b/reference_model/src/generate/generate_utils.cc
index cf5308b..8b16e97 100644
--- a/reference_model/src/generate/generate_utils.cc
+++ b/reference_model/src/generate/generate_utils.cc
@@ -71,14 +71,15 @@
{ Op::Op_PAD, "PAD" },
{ Op::Op_POW, "POW" },
{ Op::Op_RECIPROCAL, "RECIPROCAL" },
- { Op::Op_RESHAPE, "RESHAPE" },
- { Op::Op_RSQRT, "RSQRT" },
{ Op::Op_REDUCE_MAX, "REDUCE_MAX" },
{ Op::Op_REDUCE_MIN, "REDUCE_MIN" },
{ Op::Op_REDUCE_PRODUCT, "REDUCE_PRODUCT" },
{ Op::Op_REDUCE_SUM, "REDUCE_SUM" },
+ { Op::Op_RESHAPE, "RESHAPE" },
+ { Op::Op_RESIZE, "RESIZE" },
{ Op::Op_REVERSE, "REVERSE" },
{ Op::Op_RFFT2D, "RFFT2D" },
+ { Op::Op_RSQRT, "RSQRT" },
{ Op::Op_SCATTER, "SCATTER" },
{ Op::Op_SELECT, "SELECT" },
{ Op::Op_SIGMOID, "SIGMOID" },
diff --git a/reference_model/src/verify/verifiers.h b/reference_model/src/verify/verifiers.h
index 6830115..80b6e19 100644
--- a/reference_model/src/verify/verifiers.h
+++ b/reference_model/src/verify/verifiers.h
@@ -71,6 +71,17 @@
/// \return True if compliant else false
bool verifyAbsError(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const AbsErrorVerifyInfo& aeInfo);
+/// \brief Perform relative result verification
+///
+/// \param referenceTensor Reference tensor
+/// \param implementationTensor Implementation resulting tensor
+/// \param rInfo Relative verification meta-data
+///
+/// \return True if compliant else false
+bool verifyRelative(const CTensor* referenceTensor,
+ const CTensor* implementationTensor,
+ const RelativeVerifyInfo& rInfo);
+
}; // namespace TosaReference
#endif // VERIFIERS_H_
diff --git a/reference_model/src/verify/verify_entry.cc b/reference_model/src/verify/verify_entry.cc
index afc5916..9702c36 100644
--- a/reference_model/src/verify/verify_entry.cc
+++ b/reference_model/src/verify/verify_entry.cc
@@ -43,6 +43,9 @@
case VerifyMode::AbsError: {
return verifyAbsError(ref, refBnd, imp, cfg.absErrorInfo);
}
+ case VerifyMode::Relative: {
+ return verifyRelative(ref, imp, cfg.relativeInfo);
+ }
default: {
WARNING("[Verifier] Unsupported verification mode.");
break;
diff --git a/reference_model/src/verify/verify_relative.cc b/reference_model/src/verify/verify_relative.cc
new file mode 100644
index 0000000..b12daf7
--- /dev/null
+++ b/reference_model/src/verify/verify_relative.cc
@@ -0,0 +1,83 @@
+
+// Copyright (c) 2024, ARM Limited.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <vector>
+
+#include "verifiers.h"
+#include "verify/verify_utils.h"
+
+namespace TosaReference
+{
+
+namespace
+{
+template <typename OutDtype>
+bool validateData(const double* ref,
+ const OutDtype* imp,
+ const std::vector<int32_t>& shape,
+ const RelativeVerifyInfo& cfg)
+{
+ const size_t T = static_cast<size_t>(numElements(shape));
+ TOSA_REF_REQUIRE(T > 0, "[R] Invalid shape for reference tensor");
+
+ double errBound = cfg.max * cfg.scale;
+ for (size_t i = 0; i < T; ++i)
+ {
+ bool valid = tosaCheckFloatBound(imp[i], ref[i], errBound);
+ if (!valid)
+ {
+ auto pos = indexToPosition(i, shape);
+ WARNING("[Verifier][RP] Location %s", positionToString(pos).c_str());
+ return false;
+ }
+ }
+ return true;
+}
+} // namespace
+
+bool verifyRelative(const CTensor* referenceTensor,
+ const CTensor* implementationTensor,
+ const RelativeVerifyInfo& rInfo)
+{
+ // Validate that tensors are provided
+ TOSA_REF_REQUIRE(referenceTensor != nullptr, "[R] Reference tensor is missing");
+ TOSA_REF_REQUIRE(implementationTensor != nullptr, "[R] Implementation tensor is missing");
+
+ const std::vector<int32_t> refShape(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims);
+
+ const double* refData = reinterpret_cast<const double*>(referenceTensor->data);
+ TOSA_REF_REQUIRE(refData != nullptr, "[R] Missing data for reference");
+
+ switch (implementationTensor->data_type)
+ {
+ case tosa_datatype_fp32_t: {
+ const auto* impData = reinterpret_cast<const float*>(implementationTensor->data);
+ TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation");
+ return validateData(refData, impData, refShape, rInfo);
+ }
+ case tosa_datatype_fp16_t: {
+ const auto* impData = reinterpret_cast<const half_float::half*>(implementationTensor->data);
+ TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation");
+ return validateData(refData, impData, refShape, rInfo);
+ }
+ default:
+ WARNING("[Verifier][R] Data-type not supported.");
+ break;
+ }
+
+ return false;
+}
+} // namespace TosaReference
diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc
index abb55eb..14bc6f1 100644
--- a/reference_model/src/verify/verify_utils.cc
+++ b/reference_model/src/verify/verify_utils.cc
@@ -52,6 +52,7 @@
{ VerifyMode::FpSpecial, "FP_SPECIAL" },
{ VerifyMode::ReduceProduct, "REDUCE_PRODUCT" },
{ VerifyMode::AbsError, "ABS_ERROR" },
+ { VerifyMode::Relative, "RELATIVE" },
})
void from_json(const nlohmann::json& j, UlpVerifyInfo& ulpInfo)
@@ -78,6 +79,12 @@
}
}
+void from_json(const nlohmann::json& j, RelativeVerifyInfo& rInfo)
+{
+ j.at("max").get_to(rInfo.max);
+ j.at("scale").get_to(rInfo.scale);
+}
+
void from_json(const nlohmann::json& j, VerifyConfig& cfg)
{
j.at("mode").get_to(cfg.mode);
@@ -100,6 +107,10 @@
{
j.at("abs_error_info").get_to(cfg.absErrorInfo);
}
+ if (j.contains("relative_info"))
+ {
+ j.at("relative_info").get_to(cfg.relativeInfo);
+ }
}
std::optional<VerifyConfig> parseVerifyConfig(const char* tensorName, const char* json)
diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h
index 0fc68fb..341bd90 100644
--- a/reference_model/src/verify/verify_utils.h
+++ b/reference_model/src/verify/verify_utils.h
@@ -46,7 +46,8 @@
DotProduct,
FpSpecial,
ReduceProduct,
- AbsError
+ AbsError,
+ Relative
};
/// \brief ULP verification meta-data
@@ -83,6 +84,15 @@
double lowerBound;
};
+/// \brief relative verification meta-data
+struct RelativeVerifyInfo
+{
+ RelativeVerifyInfo() = default;
+
+ double max;
+ double scale;
+};
+
/// \brief Verification meta-data
struct VerifyConfig
{
@@ -94,6 +104,7 @@
DotProductVerifyInfo dotProductInfo;
ReduceProductVerifyInfo reduceProductInfo;
AbsErrorVerifyInfo absErrorInfo;
+ RelativeVerifyInfo relativeInfo;
};
/// \brief Parse the verification config for a tensor when given in JSON form
diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp
index ba18af1..a85546e 100644
--- a/reference_model/test/verify_tests.cpp
+++ b/reference_model/test/verify_tests.cpp
@@ -29,6 +29,16 @@
namespace
{
+void update_json_template(std::string& str, const std::string& find, const std::string& change)
+{
+ // Update the 'str' by looking for instances of 'find' and replacing them with 'change'
+ auto pos = str.find(find);
+ while (pos != std::string::npos)
+ {
+ str.replace(pos, find.length(), change);
+ pos = str.find(find);
+ }
+}
class TosaTensor
{
@@ -472,7 +482,7 @@
SUBCASE("outside")
{
- // Generate some data that exceeds a specified number of ULP for each value in the tensor.
+ // Generate some data that exceeds a requirements for each value in the tensor.
auto otherData_fp32 = data_fp32;
std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [outsideErrBound](auto& value) {
if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value))
@@ -489,4 +499,76 @@
jsonCfg.c_str()));
}
}
+
+TEST_CASE("positive - relative")
+{
+ std::string templateJsonCfg = R"({
+ "tensors" : {
+ "out1" : {
+ "mode": "RELATIVE",
+ "data_type": "FP32",
+ "relative_info": {
+ "max": _MAXIMUM_,
+ "scale": _SCALE_
+ }
+ }
+ }
+ })";
+
+ const auto shape = std::vector<int32_t>{ 3, 3, 3 };
+ const auto elementCount = std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>());
+
+ // Generate some random floats using the full range of fp32.
+ auto data_fp32 = generateRandomTensorData<float>(elementCount, true);
+ std::vector<double> data_fp64(data_fp32.begin(), data_fp32.end());
+
+ float scale = 0.0006;
+ float max = 0.0;
+ std::for_each(std::begin(data_fp32), std::end(data_fp32), [&max](auto& value) {
+ if (!std::isinf(value) && !std::isnan(value))
+ {
+ max = std::max(max, std::abs(value));
+ }
+ });
+ std::string jsonCfg = templateJsonCfg;
+ update_json_template(jsonCfg, "_MAXIMUM_", std::to_string(max));
+ update_json_template(jsonCfg, "_SCALE_", std::to_string(scale));
+
+ float errBound = max * scale;
+ // Use 10% error margin to test due to using v.large values in our random data
+ float insideErrBound = errBound * 0.9;
+ float outsideErrBound = errBound * 1.1;
+
+ SUBCASE("inside")
+ {
+ // Generate some data that meets the requirements of the result.
+ auto otherData_fp32 = data_fp32;
+ std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [insideErrBound](auto& value) {
+ if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value))
+ value += insideErrBound;
+ });
+ const auto referenceTensor =
+ TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data_fp64.data()));
+ const auto implementationTensor =
+ TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData_fp32.data()));
+ REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
+ }
+
+ SUBCASE("outside")
+ {
+ // Generate some data that exceeds the requirements for each value in the tensor.
+ auto otherData_fp32 = data_fp32;
+ std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [outsideErrBound](auto& value) {
+ if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value))
+ value += outsideErrBound;
+ });
+
+ const auto referenceTensor =
+ TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data_fp64.data()));
+ const auto implementationTensor =
+ TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData_fp32.data()));
+ REQUIRE_FALSE(
+ tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
+ }
+}
TEST_SUITE_END(); // verify
diff --git a/scripts/schemavalidation/compliance-config.schema.json b/scripts/schemavalidation/compliance-config.schema.json
index c0a479d..eb3ccde 100644
--- a/scripts/schemavalidation/compliance-config.schema.json
+++ b/scripts/schemavalidation/compliance-config.schema.json
@@ -88,7 +88,30 @@
"required": [
"n"
]
+ },
+ "relative_info": {
+ "description": "info required for the RELATIVE mode",
+ "type": "object",
+ "properties":
+ {
+ "max": {
+ "description": "maximum absolute input value",
+ "type": "number",
+ "minimum": 0
+ },
+ "scale": {
+ "description": "relative scaling factor",
+ "type": "number",
+ "minimum": 0
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "max",
+ "scale"
+ ]
}
+
},
"additionalProperties": false,
"required": [
diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json
index b8efd35..dc28bef 100644
--- a/verif/conformance/tosa_main_profile_ops_info.json
+++ b/verif/conformance/tosa_main_profile_ops_info.json
@@ -2645,6 +2645,7 @@
"profile": [
"tosa-mi"
],
+ "support_for": [ "lazy_data_gen" ],
"generation": {
"standard": {
"negative_dim_range": "1,10",
@@ -2657,13 +2658,13 @@
"--target-dtype",
"bf16",
"--fp-values-range",
- "-2.0,2.0"
+ "-max,max"
],
[
"--target-dtype",
"fp32",
"--fp-values-range",
- "-2.0,2.0",
+ "-max,max",
"--target-shape",
"1,1103,1,2",
"--max-resize-output-dim",
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index a4bced3..4630f35 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -1394,6 +1394,22 @@
testGen, opName, dtypeList, shapeList, argsDict, error_name
)
+ @staticmethod
+ def tvgResize(testGen, opName, dtypeList, shapeList, argsDict, error_name=None):
+ data_range = TosaTensorValuesGen._get_data_range(
+ testGen,
+ dtypeList[0],
+ TosaTensorValuesGen.TVG_FLOAT_HIGH_VALUE,
+ )
+ if data_range:
+ argsDict["data_range"] = data_range
+ # Needed for compliance
+ argsDict["max_abs_value"] = data_range[1]
+
+ return TosaTensorValuesGen.tvgLazyGenDefault(
+ testGen, opName, dtypeList, shapeList, argsDict, error_name
+ )
+
# Set the POW exponent high data range
TVG_FLOAT_HIGH_VALUE_POW_EXP = {
DType.FP32: 10.0,
@@ -3343,14 +3359,13 @@
border[0],
border[1],
),
- [
- mode,
- scale,
- offset,
- border,
- dtype,
- outputDTypeNew,
- ],
+ {
+ "mode": mode,
+ "scale": scale,
+ "offset": offset,
+ "border": border,
+ "output_dtype": outputDTypeNew,
+ },
)
if arg_to_append in arg_list:
# Skip already generated test params
@@ -3359,6 +3374,16 @@
# Valid permutation
perm += 1
arg_list.append(arg_to_append)
+
+ # Now add data generator types
+ arg_list = TosaArgGen._add_data_generators(
+ testGen,
+ opName,
+ dtype,
+ arg_list,
+ error_name,
+ )
+ # Return list of tuples: (arg_str, args_dict)
return arg_list
@staticmethod
diff --git a/verif/generator/tosa_error_if.py b/verif/generator/tosa_error_if.py
index 90c3428..5fd647a 100644
--- a/verif/generator/tosa_error_if.py
+++ b/verif/generator/tosa_error_if.py
@@ -2585,9 +2585,9 @@
@staticmethod
def ivWrongDataTypeOrModeResize(**kwargs):
input_dtype = kwargs["input_dtype"]
- args = kwargs["args"]
- mode = args[0]
- output_dtype = args[5]
+ args_dict = kwargs["args"]
+ mode = args_dict["mode"]
+ output_dtype = args_dict["output_dtype"]
if mode == ResizeMode.BILINEAR:
# Invalid output data type / Invalid input datatype
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index d82f919..ae689b4 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -357,6 +357,12 @@
elif "compliance" in op and "ulp" in op["compliance"]:
mode = gtu.ComplianceMode.ULP
compliance_tens["ulp_info"] = {"ulp": op["compliance"]["ulp"]}
+ elif "compliance" in op and "relative" in op["compliance"]:
+ mode = gtu.ComplianceMode.RELATIVE
+ compliance_tens["relative_info"] = {
+ "max": argsDict["max_abs_value"],
+ "scale": op["compliance"]["relative"],
+ }
elif op["op"] == Op.REDUCE_PRODUCT:
mode = gtu.ComplianceMode.REDUCE_PRODUCT
compliance_tens["reduce_product_info"] = {"n": argsDict["n"]}
@@ -1933,17 +1939,21 @@
def build_resize(
self,
op,
- input,
- mode,
- scale,
- offset,
- border,
- input_dtype,
- output_dtype,
+ inputs,
+ args_dict,
validator_fcns,
error_name=None,
+ qinfo=None,
):
- result_tens = OutputShaper.resizeOp(
+ assert len(inputs) == 1
+ input = inputs[0]
+ mode = args_dict["mode"]
+ scale = args_dict["scale"]
+ offset = args_dict["offset"]
+ border = args_dict["border"]
+ output_dtype = args_dict["output_dtype"]
+
+ result_tensor = OutputShaper.resizeOp(
self.ser,
self.rng,
input,
@@ -1951,14 +1961,14 @@
scale,
offset,
border,
- input_dtype,
+ input.dtype,
output_dtype,
error_name,
)
# Invalidate Input/Output list for error if checks.
input_list = [input.name]
- output_list = [result_tens.name]
+ output_list = [result_tensor.name]
pCount, cCount = op["operands"]
num_operands = pCount + cCount
input_list, output_list = TosaErrorIfArgGen.eiInvalidateInputOutputList(
@@ -1972,25 +1982,28 @@
op=op,
mode=mode,
scale=scale,
- input_dtype=input_dtype,
+ input_dtype=input.dtype,
output_dtype=output_dtype,
input_shape=input.shape,
- output_shape=result_tens.shape,
+ output_shape=result_tensor.shape,
offset=offset,
border=border,
input_list=input_list,
output_list=output_list,
- result_tensors=[result_tens],
+ result_tensors=[result_tensor],
num_operands=num_operands,
):
return None
attr = ts.TosaSerializerAttribute()
-
attr.ResizeAttribute(scale, offset, border, mode)
-
self.ser.addOperator(op["op"], input_list, output_list, attr)
- return result_tens
+
+ compliance = self.tensorComplianceMetaData(
+ op, input.dtype, args_dict, result_tensor, error_name
+ )
+
+ return TosaTestGen.BuildInfo(result_tensor, compliance)
def build_identityn(self, op, val, val2, validator_fcns=None, error_name=None):
result_tens = OutputShaper.unaryOp(self.ser, self.rng, val, error_name)
@@ -4610,7 +4623,7 @@
"build_fcn": (
build_resize,
TosaTensorGen.tgNHWC,
- TosaTensorValuesGen.tvgDefault,
+ TosaTensorValuesGen.tvgResize,
TosaArgGen.agResize,
),
"types": (DType.INT8, DType.INT16, DType.FP16, DType.BF16, DType.FP32),
@@ -4636,6 +4649,10 @@
TosaErrorValidator.evResizeOutputShapeMismatch,
TosaErrorValidator.evResizeOutputShapeNonInteger,
),
+ "data_gen": {
+ "fp": (gtu.DataGenType.PSEUDO_RANDOM,),
+ },
+ "compliance": {"relative": 0.006},
},
# Type conversion
"cast": {
diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py
index 6387d06..76e7388 100644
--- a/verif/generator/tosa_utils.py
+++ b/verif/generator/tosa_utils.py
@@ -39,6 +39,7 @@
FP_SPECIAL = 3
REDUCE_PRODUCT = 4
ABS_ERROR = 5
+ RELATIVE = 6
class DataGenType(IntEnum):