Add reduce product verifier
* Add verifiers to validate the result of a reduce produce operation.
* Add test cases for the new validator.
Change-Id: I666d1a67f498e7893e0f224bc5408a4134f2ef6c
Signed-off-by: Jack Frankland <jack.frankland@arm.com>
diff --git a/reference_model/CMakeLists.txt b/reference_model/CMakeLists.txt
index 9392221..cc2a5e3 100644
--- a/reference_model/CMakeLists.txt
+++ b/reference_model/CMakeLists.txt
@@ -74,6 +74,7 @@
src/verify/verify_dot_product.cc
src/verify/verify_entry.cc
src/verify/verify_exact.cc
+ src/verify/verify_reduce_product.cc
src/verify/verify_ulp.cc
src/verify/verify_utils.cc
src/ops/op_factory.cc
@@ -144,6 +145,7 @@
src/verify/verify_dot_product.cc
src/verify/verify_entry.cc
src/verify/verify_exact.cc
+ src/verify/verify_reduce_product.cc
src/verify/verify_ulp.cc
src/verify/verify_utils.cc
src/verify/verify_config.cc
diff --git a/reference_model/src/verify/verifiers.h b/reference_model/src/verify/verifiers.h
index bdc8fe7..dd97122 100644
--- a/reference_model/src/verify/verifiers.h
+++ b/reference_model/src/verify/verifiers.h
@@ -41,6 +41,16 @@
/// \return True if compliant else false
bool verifyExact(const CTensor* referenceTensor, const CTensor* implementationTensor);
+/// \brief Perform reduce product result verification
+///
+/// \param referenceTensor Reference tensor
+/// \param implementationTensor Implementation resulting tensor
+/// \param m Number of manisa bits in the floating point representation
+/// \param n Number of elements in the product
+///
+/// \return True if compliant else false
+bool verifyReduceProduct(const CTensor* referenceTensor, const CTensor* implementationTensor, uint64_t m, uint64_t n);
+
/// \brief Perform ULP result verification
///
/// \param referenceTensor Reference tensor
diff --git a/reference_model/src/verify/verify_entry.cc b/reference_model/src/verify/verify_entry.cc
index 32614b6..1ddd52b 100644
--- a/reference_model/src/verify/verify_entry.cc
+++ b/reference_model/src/verify/verify_entry.cc
@@ -34,6 +34,9 @@
case VerifyMode::Exact: {
return verifyExact(ref, imp);
}
+ case VerifyMode::ReduceProduct: {
+ return verifyReduceProduct(ref, imp, cfg.reduceProductInfo.m, cfg.reduceProductInfo.n);
+ }
case VerifyMode::Ulp: {
return verifyULP(ref, imp, cfg.ulpInfo.ulp);
}
diff --git a/reference_model/src/verify/verify_reduce_product.cc b/reference_model/src/verify/verify_reduce_product.cc
new file mode 100644
index 0000000..d233b0f
--- /dev/null
+++ b/reference_model/src/verify/verify_reduce_product.cc
@@ -0,0 +1,88 @@
+
+// Copyright (c) 2023, ARM Limited.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <vector>
+
+#include "verifiers.h"
+#include "verify/verify_utils.h"
+
+namespace
+{
+
+auto calculateError(uint64_t M, uint64_t N)
+{
+ return std::pow(1 + std::pow(2, -static_cast<int64_t>(M) - 1), N) - 1;
+}
+
+template <typename FP>
+auto calculateTolerance(uint64_t M, uint64_t N, FP value)
+{
+ return std::abs(value) * calculateError(M, N);
+}
+} // namespace
+
+namespace TosaReference
+{
+
+bool verifyReduceProduct(const CTensor* referenceTensor, const CTensor* implementationTensor, uint64_t m, uint64_t n)
+{
+ // Validate that tensors are provided
+ TOSA_REF_REQUIRE(referenceTensor != nullptr, "reference tensor is missing");
+ TOSA_REF_REQUIRE(implementationTensor != nullptr, "implementation tensor is missing");
+
+ // Get number of elements
+ const auto elementCount =
+ numElements(std::vector<int32_t>(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims));
+ TOSA_REF_REQUIRE(elementCount > 0, "invalid shape for reference tensor");
+
+ switch (implementationTensor->data_type)
+ {
+ case tosa_datatype_fp32_t: {
+ const auto* refData = reinterpret_cast<const float*>(referenceTensor->data);
+ TOSA_REF_REQUIRE(refData != nullptr, "missing data for reference");
+
+ const auto* impData = reinterpret_cast<const float*>(implementationTensor->data);
+ TOSA_REF_REQUIRE(impData != nullptr, "missing data for implementation");
+
+ return std::equal(refData, std::next(refData, elementCount), impData, std::next(impData, elementCount),
+ [m, n](const auto& referenceValue, const auto& implementationValue) {
+ // Result overflows must be set to zero of the correct sign.
+ if (std::isinf(implementationValue))
+ {
+ return implementationValue == referenceValue;
+ }
+
+ // Result underflows must be set to a zero of the correct sign.
+ if (implementationValue == 0.f || implementationValue == -0.f)
+ {
+ return implementationValue == referenceValue;
+ }
+
+ // Otherwise we are in the normal range.
+ const auto absoulteError = (referenceValue < implementationValue)
+ ? implementationValue - referenceValue
+ : referenceValue - implementationValue;
+ return absoulteError <= calculateTolerance(m, n, implementationValue);
+ });
+ }
+ default:
+ WARNING("tosa verifier: data-type not supported.");
+ break;
+ }
+
+ return false;
+}
+} // namespace TosaReference
diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc
index 786ab40..366238b 100644
--- a/reference_model/src/verify/verify_utils.cc
+++ b/reference_model/src/verify/verify_utils.cc
@@ -62,6 +62,12 @@
j.at("ks").get_to(dotProductInfo.ks);
}
+void from_json(const nlohmann::json& j, ReduceProductVerifyInfo& reduceProduceInfo)
+{
+ j.at("m").get_to(reduceProduceInfo.m);
+ j.at("n").get_to(reduceProduceInfo.n);
+}
+
void from_json(const nlohmann::json& j, VerifyConfig& cfg)
{
j.at("mode").get_to(cfg.mode);
@@ -74,6 +80,10 @@
{
j.at("dot_product_info").get_to(cfg.dotProductInfo);
}
+ if (j.contains("reduce_product_info"))
+ {
+ j.at("reduce_product_info").get_to(cfg.reduceProductInfo);
+ }
}
std::optional<VerifyConfig> parseVerifyConfig(const char* tensorName, const char* json)
diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h
index 5b98f5c..0afd804 100644
--- a/reference_model/src/verify/verify_utils.h
+++ b/reference_model/src/verify/verify_utils.h
@@ -64,6 +64,15 @@
int32_t ks;
};
+/// \brief reduce-product verification meta-data
+struct ReduceProductVerifyInfo
+{
+ ReduceProductVerifyInfo() = default;
+
+ int64_t m;
+ int64_t n;
+};
+
/// \brief Verification meta-data
struct VerifyConfig
{
@@ -73,6 +82,7 @@
DType dataType;
UlpInfo ulpInfo;
DotProductVerifyInfo dotProductInfo;
+ ReduceProductVerifyInfo reduceProductInfo;
};
/// \brief Parse the verification config for a tensor when given in JSON form
diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp
index 7b6ba9d..b75ddec 100644
--- a/reference_model/test/verify_tests.cpp
+++ b/reference_model/test/verify_tests.cpp
@@ -125,6 +125,26 @@
return data;
}
+// Calculates the "error" in the tolerance calculation as: E = pow(1 + pow(2, -M-1), N) - 1.
+// where M is the number of mantisa bits in the floating point representation and N is the number
+// of elements in the product.
+constexpr auto reduceProductError(uint64_t M, uint64_t N)
+{
+ return std::pow(1 + std::pow(2, -static_cast<int64_t>(M) - 1), N) - 1;
+}
+
+template <typename FP>
+auto reduceProductTolerance(uint64_t M, uint64_t N, const std::vector<FP>& results)
+{
+ const auto error = reduceProductError(M, N);
+ auto tolerances = std::vector<FP>(results.size());
+ for (unsigned i = 0, end = results.size(); i < end; ++i)
+ {
+ tolerances[i] = std::abs(results[i]) * error;
+ }
+ return tolerances;
+}
+
} // namespace
TEST_SUITE_BEGIN("verify");
@@ -238,6 +258,88 @@
}
}
+TEST_CASE("positive - reduce product")
+{
+ std::string json_cfg = R"({
+ "tensors" : {
+ "out1" : {
+ "mode": "REDUCE_PRODUCT",
+ "reduce_product_info": {
+ "m": 23,
+ "n": 8
+ }
+ }
+ }
+ })";
+
+ const auto inputShape = std::vector<int32_t>{ 8, 8, 8 };
+ const auto outputShape = std::vector<int32_t>{ 8, 8, 1 };
+ const auto reductionSize = inputShape[2];
+ const auto elementCount = std::accumulate(std::begin(inputShape), std::end(inputShape), 1, std::multiplies<>());
+
+ // Generate some random floats using the full range of fp32. This will be the "result" of our
+ // dot product. Here we "reduced" over the z-axis of our shape.
+ auto data = generateRandomTensorData<float>(elementCount / reductionSize, false);
+ // Calculate the tolerances for each element in the result.
+ // A float has 23 bit dedicated to the fraction.
+ constexpr uint64_t mantisa_count = 23;
+ const auto tolerances = reduceProductTolerance(mantisa_count, reductionSize, data);
+
+ SUBCASE("same")
+ {
+ // TODO: Generate some new floats that are as far away as possible from each result without
+ // exceeding the tolerance.
+ auto otherData = std::vector<float>(elementCount / reductionSize);
+ for (unsigned i = 0; i < data.size(); ++i)
+ {
+ auto newValue = data[i];
+ auto oldValue = newValue;
+ const auto target = tolerances[i] + newValue;
+
+ // Here we just increment the value until we exceed the tolerance. For simplicity we go up.
+ while (newValue < target)
+ {
+ oldValue = newValue;
+ newValue = std::nextafter(newValue, std::numeric_limits<float>::infinity());
+ }
+
+ otherData[i] = oldValue;
+ }
+
+ const auto referenceTensor =
+ TosaTensor("out1", tosa_datatype_fp64_t, outputShape, reinterpret_cast<uint8_t*>(data.data()));
+ const auto implementationTensor =
+ TosaTensor("out1", tosa_datatype_fp32_t, outputShape, reinterpret_cast<uint8_t*>(otherData.data()));
+ REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str()));
+ }
+
+ SUBCASE("different")
+ {
+ // TODO: Generate some new floats that exceed the tolerance.
+ auto otherData = std::vector<float>(elementCount / reductionSize);
+ for (unsigned i = 0; i < data.size(); ++i)
+ {
+ auto newValue = data[i];
+ const auto target = tolerances[i] + newValue;
+
+ // Here we just increment the value until we exceed the tolerance. For simplicity we go up.
+ while (newValue < target)
+ {
+ newValue = std::nextafter(newValue, std::numeric_limits<float>::infinity());
+ }
+
+ otherData[i] = newValue;
+ }
+
+ const auto referenceTensor =
+ TosaTensor("out1", tosa_datatype_fp64_t, outputShape, reinterpret_cast<uint8_t*>(data.data()));
+ const auto implementationTensor =
+ TosaTensor("out1", tosa_datatype_fp32_t, outputShape, reinterpret_cast<uint8_t*>(otherData.data()));
+ REQUIRE_FALSE(
+ tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str()));
+ }
+}
+
TEST_CASE("positive - ulp")
{
std::string json_cfg = R"({