Compliance mode testing for CONV2D
Added CONV2D data generation.
Updated verify dot product check to latest specification.
Updated test generator and python datagenerator library to create
const files during test generation.
Add support for compliance test sets to conformance test_select.
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I5be3b761a1e3ef259c058e493877cd5a89d5778b
diff --git a/reference_model/src/generate/generate_dot_product.cc b/reference_model/src/generate/generate_dot_product.cc
index cbfac4b..e6815ad 100644
--- a/reference_model/src/generate/generate_dot_product.cc
+++ b/reference_model/src/generate/generate_dot_product.cc
@@ -76,6 +76,119 @@
return true;
}
+//---------------------------------------------------------------------------//
+// Conv2D //
+//---------------------------------------------------------------------------//
+
+bool generateConv2DInput(const TosaReference::GenerateConfig& cfg,
+ TosaReference::IDotProductGenerator& generator,
+ void* data,
+ size_t size)
+{
+ if (cfg.dotProductInfo.kernel.size() != 2 || cfg.dotProductInfo.kernel[0] <= 0 || cfg.dotProductInfo.kernel[1] <= 0)
+ {
+ WARNING("[Generator][DP][Conv2D][Input] Missing or incorrect kernel size information.");
+ return false;
+ }
+ if (cfg.shape.size() != 4)
+ {
+ WARNING("[Generator][DP][Conv2D][Input] Tensor shape expected 4 dimensions.");
+ return false;
+ }
+
+ float* input = reinterpret_cast<float*>(data);
+ const int64_t T = TosaReference::numElementsFromShape(cfg.shape);
+ const uint32_t IH = cfg.shape[1];
+ const uint32_t IW = cfg.shape[2];
+ const uint32_t IC = cfg.shape[3];
+ const uint32_t KH = cfg.dotProductInfo.kernel[0];
+ const uint32_t KW = cfg.dotProductInfo.kernel[1];
+
+ for (int64_t t = 0; t < T; ++t)
+ {
+ uint32_t ic = t % IC;
+ uint32_t ix = (t / IC) % IW;
+ uint32_t iy = ((t / IC) / IW) % IH;
+ uint32_t k = ((iy % KH) * KW + (ix % KW)) * IC + ic;
+
+ input[t] = generator(k);
+ }
+ return true;
+}
+
+bool generateConv2DWeight(const TosaReference::GenerateConfig& cfg,
+ TosaReference::IDotProductGenerator& generator,
+ void* data,
+ size_t size)
+{
+ if (cfg.shape.size() != 4)
+ {
+ WARNING("[Generator][DP][Conv2D][Weight] Tensor shape expected 4 dimensions.");
+ return false;
+ }
+
+ float* weight = reinterpret_cast<float*>(data);
+ const int64_t T = TosaReference::numElementsFromShape(cfg.shape);
+ const uint32_t KH = cfg.shape[1];
+ const uint32_t KW = cfg.shape[2];
+ const uint32_t IC = cfg.shape[3];
+
+ for (int64_t t = 0; t < T; ++t)
+ {
+ uint32_t ic = t % IC;
+ uint32_t kx = (t / IC) % KW;
+ uint32_t ky = ((t / IC) / KW) % KH;
+ uint32_t k = (ky + KW * kx) * IC + ic;
+
+ weight[t] = generator(k);
+ }
+ return true;
+}
+
+bool generateConv2DBias(const TosaReference::GenerateConfig& cfg,
+ TosaReference::IDotProductGenerator& generator,
+ void* data,
+ size_t size)
+{
+ if (cfg.shape.size() != 1)
+ {
+ WARNING("[Generator][DP][Conv2D][Bias] Tensor shape expected 1 dimension.");
+ return false;
+ }
+
+ float* bias = reinterpret_cast<float*>(data);
+ const uint32_t T = cfg.shape[0];
+
+ for (uint32_t t = 0; t < T; ++t)
+ {
+ bias[t] = generator(2);
+ }
+ return true;
+}
+
+bool generateConv2D(const TosaReference::GenerateConfig& cfg,
+ TosaReference::IDotProductGenerator& generator,
+ void* data,
+ size_t size)
+{
+ if (cfg.dataType != DType::DType_FP32)
+ {
+ WARNING("[Generator][DP][Conv2D] Only supports FP32.");
+ return false;
+ }
+ switch (cfg.inputPos)
+ {
+ case 0:
+ return generateConv2DInput(cfg, generator, data, size);
+ case 1:
+ return generateConv2DWeight(cfg, generator, data, size);
+ case 2:
+ return generateConv2DBias(cfg, generator, data, size);
+ default:
+ WARNING("[Generator][DP][Conv2D] Invalid input tensor slot position to operator.");
+ return false;
+ }
+}
} // namespace
namespace TosaReference
@@ -95,6 +208,8 @@
{
case tosa::Op_MATMUL:
return generateMatMul(cfg, *generator, data, size);
+ case tosa::Op_CONV2D:
+ return generateConv2D(cfg, *generator, data, size);
default:
WARNING("[Generator][DP] Unsupported operator.");
return false;
diff --git a/reference_model/src/generate/generate_dot_product_states.cc b/reference_model/src/generate/generate_dot_product_states.cc
index 649e55e..53bef3a 100644
--- a/reference_model/src/generate/generate_dot_product_states.cc
+++ b/reference_model/src/generate/generate_dot_product_states.cc
@@ -242,7 +242,7 @@
if (_p != P2)
return (_B / std::sqrt(_KS + 1)) * s;
else
- return (_B * _B / (_KS + 1)) * s;
+ return 0.f;
}
private:
diff --git a/reference_model/src/generate/generate_utils.cc b/reference_model/src/generate/generate_utils.cc
index bcbf9d7..d3bb076 100644
--- a/reference_model/src/generate/generate_utils.cc
+++ b/reference_model/src/generate/generate_utils.cc
@@ -41,6 +41,7 @@
{ Op::Op_MATMUL, "MATMUL" },
{ Op::Op_MAX_POOL2D, "MAX_POOL2D" },
{ Op::Op_PAD, "PAD" },
+ { Op::Op_CONV2D, "CONV2D" },
})
} // namespace tosa
diff --git a/reference_model/src/generate/generate_utils.h b/reference_model/src/generate/generate_utils.h
index 0239e98..7c55f1d 100644
--- a/reference_model/src/generate/generate_utils.h
+++ b/reference_model/src/generate/generate_utils.h
@@ -52,7 +52,7 @@
int32_t ks;
DType accType;
int32_t axis;
- std::array<int32_t, 2> kernel;
+ std::vector<int32_t> kernel;
};
/// \brief Pseudo random generator meta-data
diff --git a/reference_model/src/verify/verify_dot_product.cc b/reference_model/src/verify/verify_dot_product.cc
index 2a1d273..233c072 100644
--- a/reference_model/src/verify/verify_dot_product.cc
+++ b/reference_model/src/verify/verify_dot_product.cc
@@ -14,6 +14,7 @@
#include "func_debug.h"
#include "verifiers.h"
+#include "verify_utils.h"
#include <cmath>
#include <numeric>
@@ -24,22 +25,9 @@
{
namespace
{
-
-// Accumulator precision
-template <typename T>
-struct AccPrecision;
-#define two_m42 1.0 / (double)(((int64_t)1) << 42) // 2^-42
-template <>
-struct AccPrecision<float>
-{
- static constexpr double precision = (double)(1 << 24);
- static constexpr double min_normal = two_m42 * two_m42 * two_m42; // 2^-126
-};
-#undef two_m42
-
// Generic element validation function
template <typename AccType, typename std::enable_if_t<std::is_floating_point_v<AccType>, int> = 0>
-std::optional<double> validateElement(double ref, double bnd, AccType imp, size_t KS)
+std::optional<double> validateElement(size_t index, double ref, double bnd, AccType imp, size_t KS)
{
double err = 0.0;
bool is_valid = true;
@@ -47,7 +35,11 @@
if (bnd == 0.0)
{
is_valid = (ref == 0.0) && (imp == 0.0);
- err = 0.0;
+ if (!is_valid)
+ {
+ WARNING("[Verifier][DP] index %d - bound is zero, but ref (%g) or imp (%f) is not.", index, ref, imp);
+ }
+ err = 0.0;
}
else if (std::isinf(static_cast<AccType>(bnd)))
{
@@ -58,11 +50,15 @@
else
{
// 0.0 < bnd < infinity
- const double bnd_norm = std::max(bnd, AccPrecision<AccType>::min_normal);
- const double imp_fp64 = static_cast<double>(imp);
- const double acc_prec_fp64 = AccPrecision<AccType>::precision;
- err = (imp_fp64 - ref) * acc_prec_fp64 / bnd_norm;
- is_valid = std::abs(err) <= KS;
+ const double out_err_bnd =
+ std::max(bnd * exp2(-1 - AccPrecision<AccType>::normal_frac), AccPrecision<AccType>::normal_min);
+ const double imp_fp64 = static_cast<double>(imp);
+ err = (imp_fp64 - ref) / out_err_bnd;
+ is_valid = std::abs(err) <= KS;
+ if (!is_valid)
+ {
+ WARNING("[Verifier][DP] index %d - out_err (%g) is not within KS (%d).", index, err, KS);
+ }
}
return is_valid ? std::optional(err) : std::nullopt;
@@ -73,7 +69,8 @@
bool validateData(const double* ref, const double* bnd, const AccType* imp, size_t T, const DotProductVerifyInfo& cfg)
{
const int32_t S = cfg.s;
- // TODO - needed for other ops - (max_value(bias_abs) > 0) ? (KS + 1) : KS
+ // NOTE: KS in the compliance config MUST have already been updated to (KS + 1) if the bias
+ // tensor is non-zero
const int32_t KS = cfg.ks;
double out_err_sum = 0.0;
@@ -81,7 +78,7 @@
for (size_t i = 0; i < T; ++i)
{
- auto out_err = validateElement<AccType>(ref[i], bnd[i], imp[i], KS);
+ auto out_err = validateElement<AccType>(i, ref[i], bnd[i], imp[i], KS);
TOSA_REF_REQUIRE(out_err, "[DP] Data required to be zero or error within range");
out_err_sum += out_err.value();
out_err_sumsq += out_err.value() * out_err.value();
@@ -89,11 +86,16 @@
if (S >= 3 && S <= 5)
{
+ const double max_bias = 2 * sqrt(KS * T);
+ out_err_sum = std::abs(out_err_sum);
// Check error bias magnitude for data sets S which are not positive biased
- TOSA_REF_REQUIRE(std::abs(out_err_sum) <= 2 * sqrt(KS * T), "[DP] Bias magnitude is out of range");
+ TOSA_REF_REQUIRE(out_err_sum <= max_bias, "[DP] Bias magnitude (%g) is out of range (%g)", out_err_sum,
+ max_bias);
}
// Check error variance magnitude
- TOSA_REF_REQUIRE(out_err_sumsq <= 0.4 * KS * T, "[DP] Error variance magnitude is out of range");
+ const double max_error = 0.4 * KS * T;
+ TOSA_REF_REQUIRE(out_err_sumsq <= max_error, "[DP] Error variance magnitude (%g) is out of range (%g)",
+ out_err_sumsq, max_error);
return true;
}
} // namespace
@@ -107,7 +109,7 @@
// Get number of dot-product elements
const int64_t T = numElements(std::vector<int32_t>(ref->shape, ref->shape + ref->num_dims));
- TOSA_REF_REQUIRE(T > 0, "invalid shape for reference tensor");
+ TOSA_REF_REQUIRE(T > 0, "[DP] Invalid shape for reference tensor");
const double* refData = reinterpret_cast<const double*>(ref->data);
const double* refBndData = reinterpret_cast<const double*>(refBnd->data);
diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc
index ee11c41..43ecbe7 100644
--- a/reference_model/src/verify/verify_utils.cc
+++ b/reference_model/src/verify/verify_utils.cc
@@ -140,4 +140,11 @@
return DType_UNKNOWN;
}
+
+// Like const_exp2 but for use during runtime
+double exp2(int32_t n)
+{
+ TOSA_REF_REQUIRE(-1022 <= n && n <= 1023, " Invalid exponent value (%d)", n);
+ return const_exp2(n);
+}
} // namespace TosaReference
diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h
index bbe4b4e..486ce19 100644
--- a/reference_model/src/verify/verify_utils.h
+++ b/reference_model/src/verify/verify_utils.h
@@ -23,10 +23,10 @@
#include <optional>
#include <vector>
-#define TOSA_REF_REQUIRE(COND, MESSAGE) \
+#define TOSA_REF_REQUIRE(COND, MESSAGE, ...) \
if (!(COND)) \
{ \
- WARNING("[Verifier]" MESSAGE "."); \
+ WARNING("[Verifier]" MESSAGE ".", ##__VA_ARGS__); \
return false; \
}
@@ -95,6 +95,38 @@
/// \brief Map API data-type to DType
DType mapToDType(tosa_datatype_t dataType);
+/// \brief Raise a value by the power of N or -N
+// For use during compile time - as no range check
+constexpr double const_exp2(int32_t n)
+{
+ double v = 1.0;
+ while (n > 0)
+ {
+ v = v * 2.0;
+ n--;
+ }
+ while (n < 0)
+ {
+ v = v / 2.0;
+ n++;
+ }
+ return v;
+}
+
+/// \brief Same as const_exp2 but with runtime range check of N
+double exp2(int32_t n);
+
+/// \brief Accuracy precision information
+template <typename T>
+struct AccPrecision;
+template <>
+struct AccPrecision<float>
+{
+ static constexpr double normal_min = const_exp2(-126);
+ static constexpr double normal_max = const_exp2(128) - const_exp2(127 - 23);
+ static constexpr int32_t normal_frac = 23;
+};
+
}; // namespace TosaReference
#endif // VERIFY_UTILS_H_
diff --git a/reference_model/test/generate_tests.cpp b/reference_model/test/generate_tests.cpp
index c24a369..6173372 100644
--- a/reference_model/test/generate_tests.cpp
+++ b/reference_model/test/generate_tests.cpp
@@ -286,6 +286,168 @@
matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "5", 1, expected);
}
}
+
+void conv2d_test_FP32(const std::string tosaName[3],
+ const size_t tosaElements[3],
+ const std::string templateJsonCfg,
+ const std::string setStr,
+ int32_t param,
+ const std::vector<uint32_t> lastExpected)
+{
+ std::string jsonCfg = templateJsonCfg;
+ update_json_template(jsonCfg, "_SET_", setStr);
+
+ std::vector<float> buffer(tosaElements[param]);
+ REQUIRE(tgd_generate_data(jsonCfg.c_str(), tosaName[param].c_str(), (void*)buffer.data(), tosaElements[param] * 4));
+ std::vector<float> last_three(buffer.end() - std::min<int>(3, buffer.size()), buffer.end());
+ check_output<float>(last_three, lastExpected);
+}
+
+TEST_CASE("positive - FP32 conv2d dot product (last 3 values)")
+{
+ std::string templateJsonCfg = R"({
+ "tensors" : {
+ "input" : {
+ "generator": "DOT_PRODUCT",
+ "data_type": "FP32",
+ "input_type": "VARIABLE",
+ "shape" : [ 1, 8, 2, 4 ],
+ "input_pos": 0,
+ "op" : "CONV2D",
+ "dot_product_info": {
+ "s": _SET_,
+ "ks": 16,
+ "acc_type": "FP32",
+ "kernel": [2, 2]
+ }
+ },
+ "weight" : {
+ "generator": "DOT_PRODUCT",
+ "data_type": "FP32",
+ "input_type": "CONSTANT",
+ "shape" : [ 2, 2, 2, 4 ],
+ "input_pos": 1,
+ "op" : "CONV2D",
+ "dot_product_info": {
+ "s": _SET_,
+ "ks": 16,
+ "acc_type": "FP32"
+ }
+ },
+ "bias" : {
+ "generator": "DOT_PRODUCT",
+ "data_type": "FP32",
+ "input_type": "CONSTANT",
+ "shape" : [ 2 ],
+ "input_pos": 2,
+ "op" : "CONV2D",
+ "dot_product_info": {
+ "s": _SET_,
+ "ks": 16,
+ "acc_type": "FP32"
+ }
+ }
+
+ }
+ })";
+
+ const std::string tosaName[3] = { "input", "weight", "bias" };
+ const size_t tosaElements[3] = { (1 * 8 * 2 * 4), (2 * 2 * 2 * 4), 2 };
+
+ SUBCASE("conv2d, set 0, param 0")
+ {
+ std::vector<uint32_t> lastExpected = { 0x0, 0xbf28bfda, 0xbe99cd47 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "0", 0, lastExpected);
+ }
+ SUBCASE("conv2d, set 0, param 1")
+ {
+ std::vector<uint32_t> lastExpected = { 0x0, 0x3f648dfd, 0xbd4cb21c };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "0", 1, lastExpected);
+ }
+ SUBCASE("conv2d, set 0, param 2")
+ {
+ std::vector<uint32_t> lastExpected = { 0x0, 0x0 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "0", 2, lastExpected);
+ }
+ SUBCASE("conv2d, set 1, param 0")
+ {
+ std::vector<uint32_t> lastExpected = { 0x5e6f0400, 0x5e2f78e5, 0x5e62318d };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "1", 0, lastExpected);
+ }
+ SUBCASE("conv2d, set 1, param 1")
+ {
+ // NOTE: Python test script produced 0x5e6960b0 - so off by 1
+ std::vector<uint32_t> lastExpected = { 0x5e6960af, 0x5e6d0ca9, 0x5e0b8561 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "1", 1, lastExpected);
+ }
+ SUBCASE("conv2d, set 1, param 2")
+ {
+ // NOTE: Python test script produced 0x7cf260d0, 0x7d355432 - so off by 1
+ std::vector<uint32_t> lastExpected = { 0x7cf260d1, 0x7d355431 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "1", 2, lastExpected);
+ }
+ SUBCASE("conv2d, set 2, param 0")
+ {
+ std::vector<uint32_t> lastExpected = { 0x3e7da8e9, 0x3df76a57, 0xbe338212 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "2", 0, lastExpected);
+ }
+ SUBCASE("conv2d, set 2, param 1")
+ {
+ std::vector<uint32_t> lastExpected = { 0x3daabbc5, 0xbe2f8909, 0xbdb806ec };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "2", 1, lastExpected);
+ }
+ SUBCASE("conv2d, set 2, param 2")
+ {
+ std::vector<uint32_t> lastExpected = { 0x0, 0x0 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "2", 2, lastExpected);
+ }
+ SUBCASE("conv2d, set 3, param 0")
+ {
+ std::vector<uint32_t> lastExpected = { 0xbee77fe5, 0x402141c5, 0xbda1b2ed };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "3", 0, lastExpected);
+ }
+ SUBCASE("conv2d, set 3, param 1")
+ {
+ // NOTE: Python test script produced 0xbe9947ac - so off by 1
+ std::vector<uint32_t> lastExpected = { 0x3f91e619, 0x3e9ac66b, 0xbe9947ad };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "3", 1, lastExpected);
+ }
+ SUBCASE("conv2d, set 3, param 2")
+ {
+ std::vector<uint32_t> lastExpected = { 0x0, 0x0 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "3", 2, lastExpected);
+ }
+ SUBCASE("conv2d, set 4, param 0")
+ {
+ std::vector<uint32_t> lastExpected = { 0xdd7e8575, 0x0, 0xde569ff3 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "4", 0, lastExpected);
+ }
+ SUBCASE("conv2d, set 4, param 1")
+ {
+ std::vector<uint32_t> lastExpected = { 0x5e2d6921, 0x5e13a014, 0x0 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "4", 1, lastExpected);
+ }
+ SUBCASE("conv2d, set 4, param 2")
+ {
+ std::vector<uint32_t> lastExpected = { 0x0, 0x0 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "4", 2, lastExpected);
+ }
+ SUBCASE("conv2d, set 5, param 0")
+ {
+ std::vector<uint32_t> lastExpected = { 0x5e719fb9, 0x5e6b329c, 0xdd7617d4 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "5", 0, lastExpected);
+ }
+ SUBCASE("conv2d, set 5, param 1")
+ {
+ std::vector<uint32_t> lastExpected = { 0xde42f57a, 0x5dd68799, 0xde2ddfcb };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "5", 1, lastExpected);
+ }
+ SUBCASE("conv2d, set 5, param 2")
+ {
+ std::vector<uint32_t> lastExpected = { 0x0, 0x0 };
+ conv2d_test_FP32(tosaName, tosaElements, templateJsonCfg, "5", 2, lastExpected);
+ }
+}
TEST_CASE("positive - pseudo random")
{
std::string templateJsonCfg = R"({