FP16 support in serialization
* Allow serialization of fp16 data
* Add package to support integrated half data-type (half_float::half), independent of native float: http://half.sourceforge.net/
* Allow passing of accumulate data-type in serialization
Signed-off-by: James Ward <james.ward@arm.com>
Change-Id: I54357f02e3776d81958228f699ea5044f2014f4b
diff --git a/include/attribute.def b/include/attribute.def
index b40a77b..ebbf024 100644
--- a/include/attribute.def
+++ b/include/attribute.def
@@ -26,26 +26,29 @@
...: variadic variables for more arguments, depending on NUM_ARGS_IN_ATTRIBUTES
*/
-DEF_ATTRIBUTE(Pool, 5,
+DEF_ATTRIBUTE(Pool, 6,
int32_t, V, pad,
int32_t, V, kernel,
int32_t, V, stride,
int32_t, S, input_zp,
- int32_t, S, output_zp)
+ int32_t, S, output_zp,
+ DType, S, accum_dtype)
-DEF_ATTRIBUTE(Conv, 5,
+DEF_ATTRIBUTE(Conv, 6,
int32_t, V, pad,
int32_t, V, stride,
int32_t, V, dilation,
int32_t, S, input_zp,
- int32_t, S, weight_zp)
+ int32_t, S, weight_zp,
+ DType, S, accum_dtype)
-DEF_ATTRIBUTE(TransposeConv, 5,
+DEF_ATTRIBUTE(TransposeConv, 6,
int32_t, V, out_pad,
int32_t, V, stride,
int32_t, V, output_shape,
int32_t, S, input_zp,
- int32_t, S, weight_zp)
+ int32_t, S, weight_zp,
+ DType, S, accum_dtype)
DEF_ATTRIBUTE(Pad, 3,
int32_t, V, padding,
@@ -106,13 +109,15 @@
DEF_ATTRIBUTE(Table, 1,
int16_t, V, table)
-DEF_ATTRIBUTE(MatMul, 2,
+DEF_ATTRIBUTE(MatMul, 3,
int32_t, S, a_zp,
- int32_t, S, b_zp)
+ int32_t, S, b_zp,
+ DType, S, accum_dtype)
-DEF_ATTRIBUTE(FullyConnected, 2,
+DEF_ATTRIBUTE(FullyConnected, 3,
int32_t, S, input_zp,
- int32_t, S, weight_zp)
+ int32_t, S, weight_zp,
+ DType, S, accum_dtype)
DEF_ATTRIBUTE(Negate, 2,
int32_t, S, input1_zp,
diff --git a/include/attribute.h b/include/attribute.h
index 93f7bc4..1178ee4 100644
--- a/include/attribute.h
+++ b/include/attribute.h
@@ -47,6 +47,7 @@
#define DEF_ARGS_VER0_S_float(V) DEF_ARGS_VER0_S_DEFAULT(V)
#define DEF_ARGS_VER0_S_bool(V) DEF_ARGS_VER0_S_DEFAULT(V)
#define DEF_ARGS_VER0_S_ResizeMode(V) DEF_ARGS_VER0_S_DEFAULT(V)
+#define DEF_ARGS_VER0_S_DType(V) DEF_ARGS_VER0_S_DEFAULT(V)
#define DEF_ARGS_VER0_S_string(V) DEF_ARGS_VER0_S_STR(V)
#define DEF_ARGS_VER0_S(T, V) DEF_ARGS_VER0_S_##T(V)
@@ -153,6 +154,7 @@
#undef DEF_ARGS_VER0_S_float
#undef DEF_ARGS_VER0_S_bool
#undef DEF_ARGS_VER0_S_ResizeMode
+#undef DEF_ARGS_VER0_S_DType
#undef DEF_ARGS_VER0_S_string
#undef DEF_ARGS_VER0_S_STR
#undef DEF_ARGS_VER0_S_DEFAULT
diff --git a/include/numpy_utils.h b/include/numpy_utils.h
index c64bc17..6a20eb3 100644
--- a/include/numpy_utils.h
+++ b/include/numpy_utils.h
@@ -24,6 +24,8 @@
#include <cstring>
#include <vector>
+#include "half.hpp"
+
class NumpyUtilities
{
public:
@@ -39,6 +41,8 @@
static NPError readFromNpyFile(const char* filename, const uint32_t elems, float* databuf);
+ static NPError readFromNpyFile(const char* filename, const uint32_t elems, half_float::half* databuf);
+
static NPError readFromNpyFile(const char* filename, const uint32_t elems, int32_t* databuf);
static NPError readFromNpyFile(const char* filename, const uint32_t elems, int64_t* databuf);
@@ -49,6 +53,9 @@
static NPError writeToNpyFile(const char* filename, const uint32_t elems, const bool* databuf);
+ static NPError
+ writeToNpyFile(const char* filename, const std::vector<int32_t>& shape, const half_float::half* databuf);
+
static NPError writeToNpyFile(const char* filename, const std::vector<int32_t>& shape, const int32_t* databuf);
static NPError writeToNpyFile(const char* filename, const uint32_t elems, const int32_t* databuf);
diff --git a/include/tosa_generated.h b/include/tosa_generated.h
index b54a324..f0d04d0 100644
--- a/include/tosa_generated.h
+++ b/include/tosa_generated.h
@@ -94,11 +94,12 @@
DType_INT48 = 7,
DType_FLOAT = 8,
DType_UINT16 = 9,
+ DType_FP16 = 10,
DType_MIN = DType_UNKNOWN,
- DType_MAX = DType_UINT16
+ DType_MAX = DType_FP16
};
-inline const DType (&EnumValuesDType())[10] {
+inline const DType (&EnumValuesDType())[11] {
static const DType values[] = {
DType_UNKNOWN,
DType_BOOL,
@@ -109,13 +110,14 @@
DType_INT32,
DType_INT48,
DType_FLOAT,
- DType_UINT16
+ DType_UINT16,
+ DType_FP16
};
return values;
}
inline const char * const *EnumNamesDType() {
- static const char * const names[11] = {
+ static const char * const names[12] = {
"UNKNOWN",
"BOOL",
"UINT8",
@@ -126,13 +128,14 @@
"INT48",
"FLOAT",
"UINT16",
+ "FP16",
nullptr
};
return names;
}
inline const char *EnumNameDType(DType e) {
- if (flatbuffers::IsOutRange(e, DType_UNKNOWN, DType_UINT16)) return "";
+ if (flatbuffers::IsOutRange(e, DType_UNKNOWN, DType_FP16)) return "";
const size_t index = static_cast<size_t>(e);
return EnumNamesDType()[index];
}
@@ -582,7 +585,8 @@
VT_KERNEL = 6,
VT_STRIDE = 8,
VT_INPUT_ZP = 10,
- VT_OUTPUT_ZP = 12
+ VT_OUTPUT_ZP = 12,
+ VT_ACCUM_DTYPE = 14
};
const flatbuffers::Vector<int32_t> *pad() const {
return GetPointer<const flatbuffers::Vector<int32_t> *>(VT_PAD);
@@ -599,6 +603,9 @@
int32_t output_zp() const {
return GetField<int32_t>(VT_OUTPUT_ZP, 0);
}
+ tosa::DType accum_dtype() const {
+ return static_cast<tosa::DType>(GetField<uint32_t>(VT_ACCUM_DTYPE, 0));
+ }
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_PAD) &&
@@ -609,6 +616,7 @@
verifier.VerifyVector(stride()) &&
VerifyField<int32_t>(verifier, VT_INPUT_ZP, 4) &&
VerifyField<int32_t>(verifier, VT_OUTPUT_ZP, 4) &&
+ VerifyField<uint32_t>(verifier, VT_ACCUM_DTYPE, 4) &&
verifier.EndTable();
}
};
@@ -632,6 +640,9 @@
void add_output_zp(int32_t output_zp) {
fbb_.AddElement<int32_t>(PoolAttribute::VT_OUTPUT_ZP, output_zp, 0);
}
+ void add_accum_dtype(tosa::DType accum_dtype) {
+ fbb_.AddElement<uint32_t>(PoolAttribute::VT_ACCUM_DTYPE, static_cast<uint32_t>(accum_dtype), 0);
+ }
explicit PoolAttributeBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
@@ -649,8 +660,10 @@
flatbuffers::Offset<flatbuffers::Vector<int32_t>> kernel = 0,
flatbuffers::Offset<flatbuffers::Vector<int32_t>> stride = 0,
int32_t input_zp = 0,
- int32_t output_zp = 0) {
+ int32_t output_zp = 0,
+ tosa::DType accum_dtype = tosa::DType_UNKNOWN) {
PoolAttributeBuilder builder_(_fbb);
+ builder_.add_accum_dtype(accum_dtype);
builder_.add_output_zp(output_zp);
builder_.add_input_zp(input_zp);
builder_.add_stride(stride);
@@ -665,7 +678,8 @@
const std::vector<int32_t> *kernel = nullptr,
const std::vector<int32_t> *stride = nullptr,
int32_t input_zp = 0,
- int32_t output_zp = 0) {
+ int32_t output_zp = 0,
+ tosa::DType accum_dtype = tosa::DType_UNKNOWN) {
auto pad__ = pad ? _fbb.CreateVector<int32_t>(*pad) : 0;
auto kernel__ = kernel ? _fbb.CreateVector<int32_t>(*kernel) : 0;
auto stride__ = stride ? _fbb.CreateVector<int32_t>(*stride) : 0;
@@ -675,7 +689,8 @@
kernel__,
stride__,
input_zp,
- output_zp);
+ output_zp,
+ accum_dtype);
}
struct ConvAttribute FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
@@ -685,7 +700,8 @@
VT_STRIDE = 6,
VT_DILATION = 8,
VT_INPUT_ZP = 10,
- VT_WEIGHT_ZP = 12
+ VT_WEIGHT_ZP = 12,
+ VT_ACCUM_DTYPE = 14
};
const flatbuffers::Vector<int32_t> *pad() const {
return GetPointer<const flatbuffers::Vector<int32_t> *>(VT_PAD);
@@ -702,6 +718,9 @@
int32_t weight_zp() const {
return GetField<int32_t>(VT_WEIGHT_ZP, 0);
}
+ tosa::DType accum_dtype() const {
+ return static_cast<tosa::DType>(GetField<uint32_t>(VT_ACCUM_DTYPE, 0));
+ }
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_PAD) &&
@@ -712,6 +731,7 @@
verifier.VerifyVector(dilation()) &&
VerifyField<int32_t>(verifier, VT_INPUT_ZP, 4) &&
VerifyField<int32_t>(verifier, VT_WEIGHT_ZP, 4) &&
+ VerifyField<uint32_t>(verifier, VT_ACCUM_DTYPE, 4) &&
verifier.EndTable();
}
};
@@ -735,6 +755,9 @@
void add_weight_zp(int32_t weight_zp) {
fbb_.AddElement<int32_t>(ConvAttribute::VT_WEIGHT_ZP, weight_zp, 0);
}
+ void add_accum_dtype(tosa::DType accum_dtype) {
+ fbb_.AddElement<uint32_t>(ConvAttribute::VT_ACCUM_DTYPE, static_cast<uint32_t>(accum_dtype), 0);
+ }
explicit ConvAttributeBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
@@ -752,8 +775,10 @@
flatbuffers::Offset<flatbuffers::Vector<int32_t>> stride = 0,
flatbuffers::Offset<flatbuffers::Vector<int32_t>> dilation = 0,
int32_t input_zp = 0,
- int32_t weight_zp = 0) {
+ int32_t weight_zp = 0,
+ tosa::DType accum_dtype = tosa::DType_UNKNOWN) {
ConvAttributeBuilder builder_(_fbb);
+ builder_.add_accum_dtype(accum_dtype);
builder_.add_weight_zp(weight_zp);
builder_.add_input_zp(input_zp);
builder_.add_dilation(dilation);
@@ -768,7 +793,8 @@
const std::vector<int32_t> *stride = nullptr,
const std::vector<int32_t> *dilation = nullptr,
int32_t input_zp = 0,
- int32_t weight_zp = 0) {
+ int32_t weight_zp = 0,
+ tosa::DType accum_dtype = tosa::DType_UNKNOWN) {
auto pad__ = pad ? _fbb.CreateVector<int32_t>(*pad) : 0;
auto stride__ = stride ? _fbb.CreateVector<int32_t>(*stride) : 0;
auto dilation__ = dilation ? _fbb.CreateVector<int32_t>(*dilation) : 0;
@@ -778,7 +804,8 @@
stride__,
dilation__,
input_zp,
- weight_zp);
+ weight_zp,
+ accum_dtype);
}
struct TransposeConvAttribute FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
@@ -788,7 +815,8 @@
VT_STRIDE = 6,
VT_OUTPUT_SHAPE = 8,
VT_INPUT_ZP = 10,
- VT_WEIGHT_ZP = 12
+ VT_WEIGHT_ZP = 12,
+ VT_ACCUM_DTYPE = 14
};
const flatbuffers::Vector<int32_t> *out_pad() const {
return GetPointer<const flatbuffers::Vector<int32_t> *>(VT_OUT_PAD);
@@ -805,6 +833,9 @@
int32_t weight_zp() const {
return GetField<int32_t>(VT_WEIGHT_ZP, 0);
}
+ tosa::DType accum_dtype() const {
+ return static_cast<tosa::DType>(GetField<uint32_t>(VT_ACCUM_DTYPE, 0));
+ }
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_OUT_PAD) &&
@@ -815,6 +846,7 @@
verifier.VerifyVector(output_shape()) &&
VerifyField<int32_t>(verifier, VT_INPUT_ZP, 4) &&
VerifyField<int32_t>(verifier, VT_WEIGHT_ZP, 4) &&
+ VerifyField<uint32_t>(verifier, VT_ACCUM_DTYPE, 4) &&
verifier.EndTable();
}
};
@@ -838,6 +870,9 @@
void add_weight_zp(int32_t weight_zp) {
fbb_.AddElement<int32_t>(TransposeConvAttribute::VT_WEIGHT_ZP, weight_zp, 0);
}
+ void add_accum_dtype(tosa::DType accum_dtype) {
+ fbb_.AddElement<uint32_t>(TransposeConvAttribute::VT_ACCUM_DTYPE, static_cast<uint32_t>(accum_dtype), 0);
+ }
explicit TransposeConvAttributeBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
@@ -855,8 +890,10 @@
flatbuffers::Offset<flatbuffers::Vector<int32_t>> stride = 0,
flatbuffers::Offset<flatbuffers::Vector<int32_t>> output_shape = 0,
int32_t input_zp = 0,
- int32_t weight_zp = 0) {
+ int32_t weight_zp = 0,
+ tosa::DType accum_dtype = tosa::DType_UNKNOWN) {
TransposeConvAttributeBuilder builder_(_fbb);
+ builder_.add_accum_dtype(accum_dtype);
builder_.add_weight_zp(weight_zp);
builder_.add_input_zp(input_zp);
builder_.add_output_shape(output_shape);
@@ -871,7 +908,8 @@
const std::vector<int32_t> *stride = nullptr,
const std::vector<int32_t> *output_shape = nullptr,
int32_t input_zp = 0,
- int32_t weight_zp = 0) {
+ int32_t weight_zp = 0,
+ tosa::DType accum_dtype = tosa::DType_UNKNOWN) {
auto out_pad__ = out_pad ? _fbb.CreateVector<int32_t>(*out_pad) : 0;
auto stride__ = stride ? _fbb.CreateVector<int32_t>(*stride) : 0;
auto output_shape__ = output_shape ? _fbb.CreateVector<int32_t>(*output_shape) : 0;
@@ -881,7 +919,8 @@
stride__,
output_shape__,
input_zp,
- weight_zp);
+ weight_zp,
+ accum_dtype);
}
struct PadAttribute FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
@@ -1772,7 +1811,8 @@
typedef MatMulAttributeBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_A_ZP = 4,
- VT_B_ZP = 6
+ VT_B_ZP = 6,
+ VT_ACCUM_DTYPE = 8
};
int32_t a_zp() const {
return GetField<int32_t>(VT_A_ZP, 0);
@@ -1780,10 +1820,14 @@
int32_t b_zp() const {
return GetField<int32_t>(VT_B_ZP, 0);
}
+ tosa::DType accum_dtype() const {
+ return static_cast<tosa::DType>(GetField<uint32_t>(VT_ACCUM_DTYPE, 0));
+ }
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int32_t>(verifier, VT_A_ZP, 4) &&
VerifyField<int32_t>(verifier, VT_B_ZP, 4) &&
+ VerifyField<uint32_t>(verifier, VT_ACCUM_DTYPE, 4) &&
verifier.EndTable();
}
};
@@ -1798,6 +1842,9 @@
void add_b_zp(int32_t b_zp) {
fbb_.AddElement<int32_t>(MatMulAttribute::VT_B_ZP, b_zp, 0);
}
+ void add_accum_dtype(tosa::DType accum_dtype) {
+ fbb_.AddElement<uint32_t>(MatMulAttribute::VT_ACCUM_DTYPE, static_cast<uint32_t>(accum_dtype), 0);
+ }
explicit MatMulAttributeBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
@@ -1812,8 +1859,10 @@
inline flatbuffers::Offset<MatMulAttribute> CreateMatMulAttribute(
flatbuffers::FlatBufferBuilder &_fbb,
int32_t a_zp = 0,
- int32_t b_zp = 0) {
+ int32_t b_zp = 0,
+ tosa::DType accum_dtype = tosa::DType_UNKNOWN) {
MatMulAttributeBuilder builder_(_fbb);
+ builder_.add_accum_dtype(accum_dtype);
builder_.add_b_zp(b_zp);
builder_.add_a_zp(a_zp);
return builder_.Finish();
@@ -1823,7 +1872,8 @@
typedef FullyConnectedAttributeBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_INPUT_ZP = 4,
- VT_WEIGHT_ZP = 6
+ VT_WEIGHT_ZP = 6,
+ VT_ACCUM_DTYPE = 8
};
int32_t input_zp() const {
return GetField<int32_t>(VT_INPUT_ZP, 0);
@@ -1831,10 +1881,14 @@
int32_t weight_zp() const {
return GetField<int32_t>(VT_WEIGHT_ZP, 0);
}
+ tosa::DType accum_dtype() const {
+ return static_cast<tosa::DType>(GetField<uint32_t>(VT_ACCUM_DTYPE, 0));
+ }
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int32_t>(verifier, VT_INPUT_ZP, 4) &&
VerifyField<int32_t>(verifier, VT_WEIGHT_ZP, 4) &&
+ VerifyField<uint32_t>(verifier, VT_ACCUM_DTYPE, 4) &&
verifier.EndTable();
}
};
@@ -1849,6 +1903,9 @@
void add_weight_zp(int32_t weight_zp) {
fbb_.AddElement<int32_t>(FullyConnectedAttribute::VT_WEIGHT_ZP, weight_zp, 0);
}
+ void add_accum_dtype(tosa::DType accum_dtype) {
+ fbb_.AddElement<uint32_t>(FullyConnectedAttribute::VT_ACCUM_DTYPE, static_cast<uint32_t>(accum_dtype), 0);
+ }
explicit FullyConnectedAttributeBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
@@ -1863,8 +1920,10 @@
inline flatbuffers::Offset<FullyConnectedAttribute> CreateFullyConnectedAttribute(
flatbuffers::FlatBufferBuilder &_fbb,
int32_t input_zp = 0,
- int32_t weight_zp = 0) {
+ int32_t weight_zp = 0,
+ tosa::DType accum_dtype = tosa::DType_UNKNOWN) {
FullyConnectedAttributeBuilder builder_(_fbb);
+ builder_.add_accum_dtype(accum_dtype);
builder_.add_weight_zp(weight_zp);
builder_.add_input_zp(input_zp);
return builder_.Finish();
diff --git a/include/tosa_serialization_handler.h b/include/tosa_serialization_handler.h
index 2a992b2..462c7ef 100644
--- a/include/tosa_serialization_handler.h
+++ b/include/tosa_serialization_handler.h
@@ -294,6 +294,7 @@
tosa_err_t LoadFileSchema(const char* schema_filename);
// data format conversion. little-endian.
+ static tosa_err_t ConvertF16toU8(const std::vector<float>& in, std::vector<uint8_t>& out);
static tosa_err_t ConvertF32toU8(const std::vector<float>& in, std::vector<uint8_t>& out);
static tosa_err_t ConvertI48toU8(const std::vector<int64_t>& in, std::vector<uint8_t>& out);
static tosa_err_t ConvertI32toU8(const std::vector<int32_t>& in, std::vector<uint8_t>& out);
@@ -302,6 +303,7 @@
static tosa_err_t ConvertI4toU8(const std::vector<int8_t>& in, std::vector<uint8_t>& out);
static tosa_err_t ConvertBooltoU8(const std::vector<bool>& in, std::vector<uint8_t>& out);
+ static tosa_err_t ConvertU8toF16(const std::vector<uint8_t>& in, uint32_t out_size, std::vector<float>& out);
static tosa_err_t ConvertU8toF32(const std::vector<uint8_t>& in, uint32_t out_size, std::vector<float>& out);
static tosa_err_t ConvertU8toI48(const std::vector<uint8_t>& in, uint32_t out_size, std::vector<int64_t>& out);
static tosa_err_t ConvertU8toI32(const std::vector<uint8_t>& in, uint32_t out_size, std::vector<int32_t>& out);