blob: 0ce621110eacdd7da2938b5c2637ecbf56ddba5e [file] [log] [blame]
Eric Kunze2364dcd2021-04-26 11:06:57 -07001
Tai Ly5d580fa2023-12-15 20:34:51 +00002// Copyright (c) 2020-2024, ARM Limited.
Eric Kunze2364dcd2021-04-26 11:06:57 -07003//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15
16#include "tosa_serialization_handler.h"
James Ward485a11d2022-08-05 13:48:37 +010017#include "half.hpp"
Eric Kunze2364dcd2021-04-26 11:06:57 -070018
19#include <iostream>
20using namespace tosa;
21
Won Jeon520b7ca2024-04-19 14:21:00 +000022using fp8e4m3 = ct::cfloat<int8_t, 4, true, true, false>;
23using fp8e5m2 = ct::cfloat<int8_t, 5, true, true, true>;
Tai Lyce911a22024-03-21 17:01:14 +000024
Eric Kunze2364dcd2021-04-26 11:06:57 -070025TosaSerializationTensor::TosaSerializationTensor(const flatbuffers::String* name,
Kevin Cheng3bb1bc12021-06-17 15:57:08 -070026 const flatbuffers::Vector<int32_t>* shape,
Eric Kunze2364dcd2021-04-26 11:06:57 -070027 DType dtype,
Jerry Ge442261b2022-09-09 13:38:56 -070028 const flatbuffers::Vector<uint8_t>* data,
Tai Lyc6939a42023-08-21 17:00:29 +000029 const bool variable,
Tai Lyd0520b92023-09-19 21:30:18 +000030 const bool is_unranked,
31 const flatbuffers::String* variable_name)
Eric Kunze2364dcd2021-04-26 11:06:57 -070032{
Jerry Ge442261b2022-09-09 13:38:56 -070033 _dtype = dtype;
34 _variable = variable;
Jerry Geb413a952023-05-08 19:17:22 +000035 if (shape)
36 {
37 std::copy(shape->begin(), shape->end(), std::back_inserter(_shape));
38 }
Eric Kunze2364dcd2021-04-26 11:06:57 -070039
40 assert(name);
41 _name = name->str();
42
Kevin Cheng3bb1bc12021-06-17 15:57:08 -070043 if (data)
Eric Kunze2364dcd2021-04-26 11:06:57 -070044 {
Kevin Cheng3bb1bc12021-06-17 15:57:08 -070045 std::copy(data->begin(), data->end(), std::back_inserter(_data));
Eric Kunze2364dcd2021-04-26 11:06:57 -070046 }
Tai Lyc6939a42023-08-21 17:00:29 +000047 _is_unranked = is_unranked;
Tai Lyd0520b92023-09-19 21:30:18 +000048
49 if (variable_name)
50 {
51 _variable_name = variable_name->str();
52 }
Eric Kunze2364dcd2021-04-26 11:06:57 -070053}
54
Kevin Cheng545a5082021-11-11 01:36:33 +000055TosaSerializationTensor::TosaSerializationTensor(const std::string& name,
Eric Kunze2364dcd2021-04-26 11:06:57 -070056 const std::vector<int32_t>& shape,
57 DType dtype,
Jerry Ge442261b2022-09-09 13:38:56 -070058 const std::vector<uint8_t>& data,
Tai Lyc6939a42023-08-21 17:00:29 +000059 const bool variable,
Tai Lyd0520b92023-09-19 21:30:18 +000060 const bool is_unranked,
61 const std::string& variable_name)
Eric Kunze2364dcd2021-04-26 11:06:57 -070062{
Tai Lyd0520b92023-09-19 21:30:18 +000063 _dtype = dtype;
64 _variable = variable;
65 _shape = shape;
66 _name = name;
67 _data = data;
68 _is_unranked = is_unranked;
69 _variable_name = variable_name;
Eric Kunze2364dcd2021-04-26 11:06:57 -070070}
71
72TosaSerializationTensor::TosaSerializationTensor()
73{
Tai Lyc6939a42023-08-21 17:00:29 +000074 _dtype = DType_UNKNOWN;
75 _variable = false;
76 _name = "UNKNOWN";
77 _is_unranked = false;
Eric Kunze2364dcd2021-04-26 11:06:57 -070078}
79
80TosaSerializationTensor::~TosaSerializationTensor()
81{}
82
Eric Kunzebdcc3fe2022-06-07 05:17:37 +000083void TosaSerializationOperator::InitializeAttribute(Attribute attribute_type, const TosaAttributeBase* attribute)
Eric Kunze2364dcd2021-04-26 11:06:57 -070084{
Eric Kunze2364dcd2021-04-26 11:06:57 -070085 _attribute_type = attribute_type;
86
87 switch (attribute_type)
88 {
89 case Attribute_NONE:
90 _attribute = new TosaNoneAttribute();
91 break;
92#define DEF_ATTRIBUTE(NAME, ...) \
93 case Attribute_##NAME##Attribute: \
94 _attribute = new Tosa##NAME##Attribute(attribute); \
95 break;
96#include "attribute.def"
97#undef DEF_ATTRIBUTE
98 default:
99 printf("TosaSerializationOperator::TosaSerializationOperator(): Attribute %s not implemented yet\n",
100 EnumNamesAttribute()[attribute_type]);
101 assert(0);
102 }
103
Eric Kunzebdcc3fe2022-06-07 05:17:37 +0000104 assert(_attribute);
Kevin Cheng545a5082021-11-11 01:36:33 +0000105}
Eric Kunze2364dcd2021-04-26 11:06:57 -0700106
Kevin Cheng545a5082021-11-11 01:36:33 +0000107TosaSerializationOperator::TosaSerializationOperator(Op op,
108 Attribute attribute_type,
109 const TosaAttributeBase* attribute,
Kevin Cheng545a5082021-11-11 01:36:33 +0000110 const std::vector<std::string>& input_tensor_names,
111 const std::vector<std::string>& output_tensor_names)
112{
113 _op = op;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700114 _input_tensor_names = input_tensor_names;
115 _output_tensor_names = output_tensor_names;
Kevin Cheng545a5082021-11-11 01:36:33 +0000116
Eric Kunzebdcc3fe2022-06-07 05:17:37 +0000117 InitializeAttribute(attribute_type, attribute);
Kevin Cheng545a5082021-11-11 01:36:33 +0000118}
119
120TosaSerializationOperator::TosaSerializationOperator(Op op,
121 Attribute attribute_type,
122 const TosaAttributeBase* attribute,
Kevin Cheng545a5082021-11-11 01:36:33 +0000123 std::vector<std::string>&& input_tensor_names,
124 std::vector<std::string>&& output_tensor_names)
125{
126 _op = op;
127 _input_tensor_names = std::move(input_tensor_names);
128 _output_tensor_names = std::move(output_tensor_names);
129
Eric Kunzebdcc3fe2022-06-07 05:17:37 +0000130 InitializeAttribute(attribute_type, attribute);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700131}
132
133TosaSerializationOperator::~TosaSerializationOperator()
134{
135 delete _attribute;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700136}
137
Kevin Cheng545a5082021-11-11 01:36:33 +0000138TosaSerializationBasicBlock::TosaSerializationBasicBlock(const std::string& name,
Jerry Ge13c78a62022-10-04 20:32:39 -0700139 const std::string& region_name,
Kevin Cheng545a5082021-11-11 01:36:33 +0000140 const std::vector<TosaSerializationOperator*>& operators,
141 const std::vector<TosaSerializationTensor*>& tensors,
142 const std::vector<std::string>& inputs,
143 const std::vector<std::string>& outputs)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700144{
Jerry Ge13c78a62022-10-04 20:32:39 -0700145 _name = name;
146 _region_name = region_name;
147 _operators = operators;
148 _tensors = tensors;
149 _inputs = inputs;
150 _outputs = outputs;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700151}
152
Kevin Cheng545a5082021-11-11 01:36:33 +0000153TosaSerializationBasicBlock::TosaSerializationBasicBlock(std::string&& name,
Jerry Ge13c78a62022-10-04 20:32:39 -0700154 std::string&& region_name,
Kevin Cheng545a5082021-11-11 01:36:33 +0000155 std::vector<TosaSerializationOperator*>&& operators,
156 std::vector<TosaSerializationTensor*>&& tensors,
157 std::vector<std::string>&& inputs,
158 std::vector<std::string>&& outputs)
159{
Jerry Ge13c78a62022-10-04 20:32:39 -0700160 _name = std::move(name);
161 _region_name = std::move(region_name);
162 _operators = std::move(operators);
163 _tensors = std::move(tensors);
164 _inputs = std::move(inputs);
165 _outputs = std::move(outputs);
Kevin Cheng545a5082021-11-11 01:36:33 +0000166}
167
Eric Kunze2364dcd2021-04-26 11:06:57 -0700168TosaSerializationBasicBlock::~TosaSerializationBasicBlock()
169{
170 // deallocate all operators
171 for (auto op : GetOperators())
172 {
173 delete op; // ~TosaSerializationOperator()
174 }
175
176 // deallocate all tensors
177 for (auto ts : GetTensors())
178 {
179 delete ts; // ~TosaSerializationTensor()
180 }
181}
182
Jerry Ge13c78a62022-10-04 20:32:39 -0700183TosaSerializationRegion::TosaSerializationRegion(const std::string& name,
184 const std::vector<TosaSerializationBasicBlock*>& blocks)
185{
186 _name = name;
187 _blocks = blocks;
188}
189
190TosaSerializationRegion::TosaSerializationRegion(const std::string&& name,
191 const std::vector<TosaSerializationBasicBlock*>&& blocks)
192{
193 _name = std::move(name);
194 _blocks = std::move(blocks);
195}
196
197TosaSerializationRegion::~TosaSerializationRegion()
198{
199 // deallocate all blocks
200 for (auto block : GetBlocks())
201 {
202 delete block; // ~TosaSerializationBasicBlock()
203 }
204}
205
Eric Kunze2364dcd2021-04-26 11:06:57 -0700206TosaSerializationHandler::TosaSerializationHandler()
207{
208 _schemaLoaded = false;
Kevin Chenge6563f52021-10-20 12:12:02 -0700209 _version = TosaVersion(TOSA_VERSION_MAJOR, TOSA_VERSION_MINOR, TOSA_VERSION_PATCH, TOSA_VERSION_DRAFT);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700210}
211
212TosaSerializationHandler::~TosaSerializationHandler()
213{
214 Clear(); // deallocate all basic blocks
215}
216
Eric Kunze2364dcd2021-04-26 11:06:57 -0700217tosa_err_t TosaSerializationHandler::LoadFileSchema(const char* schema_filename)
218{
219 std::string schema;
220 bool ok;
221
222 ok = flatbuffers::LoadFile(schema_filename, false, &schema);
223 if (!ok)
224 {
225 printf("Error loading schema file: %s\n", schema_filename);
226 return TOSA_FILE_ERROR;
227 }
228
229 ok = _parser.Parse(schema.c_str());
Kevin Chenga81a7a12021-11-10 14:07:34 -0800230
Eric Kunze2364dcd2021-04-26 11:06:57 -0700231 if (!ok)
232 {
233 printf("Error parsing ISA schema file: %s\n", schema_filename);
234 return TOSA_FILE_ERROR;
235 }
236 _schemaLoaded = true;
237
238 return TOSA_OK;
239}
240
241tosa_err_t TosaSerializationHandler::LoadFileJson(const char* filename)
242{
243 std::string jsonfile;
244 bool ok;
245 tosa_err_t err;
246
247 if (!_schemaLoaded)
248 {
249 return TOSA_SCHEMA_MISSING;
250 }
251
252 ok = flatbuffers::LoadFile(filename, false, &jsonfile);
253 if (!ok)
254 {
255 printf("Error loading json file: %s\n", filename);
256 return TOSA_FILE_ERROR;
257 }
258
259 ok = _parser.Parse(jsonfile.c_str());
260 if (!ok)
261 {
262 printf("Error parsing json file: %s\n", filename);
263 return TOSA_FILE_ERROR;
264 }
265
266 uint8_t* buf = _parser.builder_.GetBufferPointer();
267
Kevin Chengb97cb1d2021-10-14 11:53:39 -0700268 err = Deserialize(buf);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700269 if (err != TOSA_OK)
270 {
271 return err;
272 }
273
274 return TOSA_OK;
275}
276
277tosa_err_t TosaSerializationHandler::SaveFileJson(const char* filename)
278{
279 std::string jsongen;
280 tosa_err_t err;
281
282 if (!_schemaLoaded)
283 {
284 return TOSA_SCHEMA_MISSING;
285 }
286
Kevin Chengb97cb1d2021-10-14 11:53:39 -0700287 err = Serialize();
Eric Kunze2364dcd2021-04-26 11:06:57 -0700288 if (err != TOSA_OK)
289 {
290 return err;
291 }
292
293 uint8_t* buf = _builder.GetBufferPointer();
294
Tai Ly89963aa2023-07-03 22:14:05 +0000295 if (GenText(_parser, buf, &jsongen))
Eric Kunze2364dcd2021-04-26 11:06:57 -0700296 {
297 printf("Couldn't serialize parsed data to JSON!\n");
298 return TOSA_FILE_ERROR;
299 }
300
301 FILE* file = fopen(filename, "wb");
302
303 if (!file)
304 {
305 printf("Couldn't open output file: %s\n", filename);
306 return TOSA_FILE_ERROR;
307 }
308
309 if (fwrite(jsongen.c_str(), sizeof(char), jsongen.size(), file) != jsongen.size())
310 {
311 printf("Error writing to json output file: %s\n", filename);
312 fclose(file);
313 return TOSA_FILE_ERROR;
314 }
315
316 if (file)
317 fclose(file);
318
319 return TOSA_OK;
320}
321
322tosa_err_t TosaSerializationHandler::LoadFileTosaFlatbuffer(const char* filename)
323{
324 std::string read_buffer;
325 tosa_err_t err;
Eric Kunzeb13fe8f2022-02-17 17:14:25 -0800326 const uint8_t* buf;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700327 bool ok;
328
329 ok = flatbuffers::LoadFile(filename, false, &read_buffer);
330 if (!ok)
331 {
332 printf("Error loading flatbuffer file: %s\n", filename);
333 return TOSA_FILE_ERROR;
334 }
335
Eric Kunzeb13fe8f2022-02-17 17:14:25 -0800336 buf = reinterpret_cast<const uint8_t*>(read_buffer.data());
Eric Kunze2364dcd2021-04-26 11:06:57 -0700337
Kevin Chengb97cb1d2021-10-14 11:53:39 -0700338 err = Deserialize(buf);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700339 if (err != TOSA_OK)
340 {
341 return err;
342 }
343
344 return TOSA_OK;
345}
346
Aaron DeBattista8b3903a2021-11-18 16:38:11 +0000347tosa_err_t TosaSerializationHandler::LoadFileTosaFlatbuffer(const void* input, int in_size)
348{
349 tosa_err_t err;
350
351 const uint8_t* buf = (const uint8_t*)input;
352 err = Deserialize(buf);
353 if (err != TOSA_OK)
354 {
355 return err;
356 }
357
358 return TOSA_OK;
359}
360
Eric Kunze2364dcd2021-04-26 11:06:57 -0700361tosa_err_t TosaSerializationHandler::SaveFileTosaFlatbuffer(const char* filename)
362{
363 tosa_err_t err;
364
Kevin Chengb97cb1d2021-10-14 11:53:39 -0700365 err = Serialize();
Eric Kunze2364dcd2021-04-26 11:06:57 -0700366 if (err != TOSA_OK)
367 {
368 return err;
369 }
370
371 uint8_t* buf = _builder.GetBufferPointer();
372
373 bool ok = flatbuffers::SaveFile(filename, (const char*)buf, _builder.GetSize(), false);
374 if (!ok)
375 {
376 printf("Error saving floatbuffer file: %s\n", filename);
377 return TOSA_FILE_ERROR;
378 }
379
380 return TOSA_OK;
381}
382
383tosa_err_t TosaSerializationHandler::Clear()
384{
385 // deallocate all basic blocks
Jerry Ge13c78a62022-10-04 20:32:39 -0700386 for (auto region : GetRegions())
Eric Kunze2364dcd2021-04-26 11:06:57 -0700387 {
Jerry Ge13c78a62022-10-04 20:32:39 -0700388 delete region;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700389 }
Jerry Ge13c78a62022-10-04 20:32:39 -0700390 _regions.clear();
Eric Kunze2364dcd2021-04-26 11:06:57 -0700391
392 return TOSA_OK;
393}
394
Kevin Chengb97cb1d2021-10-14 11:53:39 -0700395tosa_err_t TosaSerializationHandler::Deserialize(const uint8_t* buf)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700396{
Eric Kunzee6596402022-06-09 21:27:36 +0000397 if (!TosaGraphBufferHasIdentifier(buf))
398 {
399 printf("WARNING: TOSA file does not have TOSA file identifier\n");
400 }
Eric Kunze2364dcd2021-04-26 11:06:57 -0700401 auto fb_tosa_graph = GetTosaGraph(buf);
402 auto fb_tosa_version = fb_tosa_graph->version();
Jerry Ge13c78a62022-10-04 20:32:39 -0700403 auto fb_tosa_regions = fb_tosa_graph->regions();
Eric Kunze2364dcd2021-04-26 11:06:57 -0700404
Eric Kunze2364dcd2021-04-26 11:06:57 -0700405 TosaAttributeBase* typed_attribute = NULL;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700406 TosaSerializationOperator* new_operator = NULL;
407 TosaSerializationBasicBlock* new_block = NULL;
408 TosaSerializationTensor* new_tensor = NULL;
Jerry Ge13c78a62022-10-04 20:32:39 -0700409 TosaSerializationRegion* new_region = NULL;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700410
411 // erase container
412 Clear();
413
Kevin Chenge6563f52021-10-20 12:12:02 -0700414 TosaVersion read_version(fb_tosa_version->_major(), fb_tosa_version->_minor(), fb_tosa_version->_patch(),
415 fb_tosa_version->_draft());
Eric Kunze2364dcd2021-04-26 11:06:57 -0700416
Jerry Gec4733b02023-08-02 21:48:39 +0000417 TosaVersion::compat_t is_compat = TosaVersion::is_compatible(read_version, GetVersion());
Kevin Chenge6563f52021-10-20 12:12:02 -0700418 switch (is_compat)
Kevin Chengb97cb1d2021-10-14 11:53:39 -0700419 {
Kevin Chenge6563f52021-10-20 12:12:02 -0700420 case TosaVersion::compat_t::COMPLETELY_COMPATIBLE:
421 break;
Jerry Gec4733b02023-08-02 21:48:39 +0000422 case TosaVersion::compat_t::BACKWARD_COMPATIBLE:
423 printf("WARNING: Different Tosa flatbuffer and serializer versions detected. Read Tosa flatbuffer version "
424 "%s is backward "
425 "compatible with serializer version %s\n",
Kevin Chenge6563f52021-10-20 12:12:02 -0700426 read_version.to_string().c_str(), GetVersion().to_string().c_str());
427 break;
428 case TosaVersion::compat_t::NOT_COMPATIBLE:
Jerry Gec4733b02023-08-02 21:48:39 +0000429 printf("ERROR: Read Tosa flatbuffer version %s is not compatible with serializer version %s\n",
Kevin Chenge6563f52021-10-20 12:12:02 -0700430 read_version.to_string().c_str(), GetVersion().to_string().c_str());
431 return TOSA_VERSION_MISMATCH;
Kevin Chengb97cb1d2021-10-14 11:53:39 -0700432 }
Eric Kunze2364dcd2021-04-26 11:06:57 -0700433
Jerry Ge13c78a62022-10-04 20:32:39 -0700434 for (size_t i = 0; i < fb_tosa_regions->size(); i++)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700435 {
Jerry Ge13c78a62022-10-04 20:32:39 -0700436 auto curr_region = fb_tosa_regions->Get(i);
437 auto region_name = curr_region->name()->str();
438 auto fb_tosa_blocks = curr_region->blocks();
Eric Kunze2364dcd2021-04-26 11:06:57 -0700439
Tai Lycfcb20d2023-03-13 21:04:11 +0000440 new_region = new TosaSerializationRegion(curr_region->name()->str(), {});
Jerry Ge13c78a62022-10-04 20:32:39 -0700441 this->GetRegions().push_back(new_region);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700442
Jerry Ge13c78a62022-10-04 20:32:39 -0700443 for (size_t i = 0; i < fb_tosa_blocks->size(); i++)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700444 {
Tai Lycfcb20d2023-03-13 21:04:11 +0000445 std::vector<TosaSerializationOperator*> block_operators_container;
446 std::vector<TosaSerializationTensor*> block_tensors_container;
447 std::vector<std::string> block_inputs_container;
448 std::vector<std::string> block_outputs_container;
449
Jerry Ge13c78a62022-10-04 20:32:39 -0700450 auto curr_block = fb_tosa_blocks->Get(i);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700451
Jerry Ge13c78a62022-10-04 20:32:39 -0700452 auto block_name = curr_block->name()->str();
Eric Kunze2364dcd2021-04-26 11:06:57 -0700453
Jerry Ge13c78a62022-10-04 20:32:39 -0700454 auto fb_tosa_operators = curr_block->operators();
Jerry Ge13c78a62022-10-04 20:32:39 -0700455 for (size_t j = 0; j < fb_tosa_operators->size(); j++)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700456 {
Jerry Ge13c78a62022-10-04 20:32:39 -0700457 auto curr_operator = fb_tosa_operators->Get(j);
458
459 auto operator_op = curr_operator->op();
460 auto attribute_type = curr_operator->attribute_type();
461 auto attribute = curr_operator->attribute();
462
Tai Lycfcb20d2023-03-13 21:04:11 +0000463 std::vector<std::string> operator_inputs_container;
464 std::vector<std::string> operator_outputs_container;
465
Jerry Ge13c78a62022-10-04 20:32:39 -0700466 // input tensors
467 auto operator_inputs = curr_operator->inputs();
Jerry Ge13c78a62022-10-04 20:32:39 -0700468 if (operator_inputs)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700469 {
Jerry Ge13c78a62022-10-04 20:32:39 -0700470 for (size_t k = 0; k < operator_inputs->size(); k++)
471 {
472 auto curr_input = operator_inputs->Get(k);
473 operator_inputs_container.push_back(curr_input->str());
474 }
Eric Kunze2364dcd2021-04-26 11:06:57 -0700475 }
Eric Kunze2364dcd2021-04-26 11:06:57 -0700476
Jerry Ge13c78a62022-10-04 20:32:39 -0700477 // output tensors
478 auto operator_outputs = curr_operator->outputs();
Jerry Ge13c78a62022-10-04 20:32:39 -0700479 if (operator_outputs)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700480 {
Jerry Ge13c78a62022-10-04 20:32:39 -0700481 for (size_t k = 0; k < operator_outputs->size(); k++)
482 {
483 auto curr_output = operator_outputs->Get(k);
484 operator_outputs_container.push_back(curr_output->str());
485 }
Eric Kunze2364dcd2021-04-26 11:06:57 -0700486 }
Eric Kunze2364dcd2021-04-26 11:06:57 -0700487
Jerry Ge13c78a62022-10-04 20:32:39 -0700488 switch (attribute_type)
489 {
490 case Attribute_NONE:
491 typed_attribute = new TosaNoneAttribute();
492 break;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700493#define DEF_ATTRIBUTE(NAME, ...) \
494 case Attribute_##NAME##Attribute: \
495 typed_attribute = new Tosa##NAME##Attribute(attribute); \
496 break;
497#include "attribute.def"
498#undef DEF_ATTRIBUTE
Jerry Ge13c78a62022-10-04 20:32:39 -0700499 default:
500 printf("TosaSerializationHandler::Deserialize(): Attribute %s not implemented yet\n",
501 EnumNamesAttribute()[attribute_type]);
502 return TOSA_INTERNAL_ERROR;
503 }
504
505 new_operator = new TosaSerializationOperator(operator_op, attribute_type, typed_attribute,
506 operator_inputs_container, operator_outputs_container);
507 if (new_operator)
508 {
509 block_operators_container.push_back(new_operator);
510 }
511 else
512 {
513 return TOSA_MEMORY_ERROR;
514 }
515
516 if (typed_attribute)
517 delete typed_attribute;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700518 }
519
Jerry Ge13c78a62022-10-04 20:32:39 -0700520 auto block_inputs = curr_block->inputs();
521 auto block_outputs = curr_block->outputs();
522
Jerry Ge13c78a62022-10-04 20:32:39 -0700523 for (size_t j = 0; j < block_inputs->size(); j++)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700524 {
Jerry Ge13c78a62022-10-04 20:32:39 -0700525 auto curr_block_input = block_inputs->Get(j);
526 block_inputs_container.push_back(curr_block_input->str());
527 }
528 for (size_t j = 0; j < block_outputs->size(); j++)
529 {
530 auto curr_block_output = block_outputs->Get(j);
531 block_outputs_container.push_back(curr_block_output->str());
532 }
533
534 auto fb_tosa_tensors = curr_block->tensors();
Jerry Ge13c78a62022-10-04 20:32:39 -0700535 for (size_t j = 0; j < fb_tosa_tensors->size(); j++)
536 {
537 auto curr_tensor = fb_tosa_tensors->Get(j);
538
Tai Ly5917fc72023-09-21 19:33:12 +0000539 auto tensor_name = curr_tensor->name();
540 auto tensor_shape = curr_tensor->shape();
541 auto tensor_type = curr_tensor->type();
542 auto tensor_variable = curr_tensor->variable();
543 auto tensor_data = curr_tensor->data();
544 auto tensor_is_unranked = curr_tensor->is_unranked();
545 auto tensor_variable_name = curr_tensor->variable_name();
Jerry Ge13c78a62022-10-04 20:32:39 -0700546
Tai Lyc6939a42023-08-21 17:00:29 +0000547 new_tensor = new TosaSerializationTensor(tensor_name, tensor_shape, tensor_type, tensor_data,
Tai Ly5917fc72023-09-21 19:33:12 +0000548 tensor_variable, tensor_is_unranked, tensor_variable_name);
Jerry Ge13c78a62022-10-04 20:32:39 -0700549 if (new_tensor)
550 {
551 block_tensors_container.push_back(new_tensor);
552 }
553 else
554 {
555 return TOSA_MEMORY_ERROR;
556 }
557 }
558 new_block = new TosaSerializationBasicBlock(block_name, region_name, block_operators_container,
559 block_tensors_container, block_inputs_container,
560 block_outputs_container);
561 if (new_block)
562 {
Tai Lycfcb20d2023-03-13 21:04:11 +0000563 new_region->GetBlocks().push_back(new_block);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700564 }
565 else
566 {
567 return TOSA_MEMORY_ERROR;
568 }
Jerry Ge13c78a62022-10-04 20:32:39 -0700569 } // end block for_loop
Eric Kunze2364dcd2021-04-26 11:06:57 -0700570 }
571
572 return TOSA_OK;
573}
574
James Ward80905bb2023-01-25 15:51:27 +0000575std::vector<uint8_t> float_to_u8_helper(float f_in)
James Wardc15f7d52022-12-07 15:38:01 +0000576{
James Ward80905bb2023-01-25 15:51:27 +0000577 // Push back a single float value to the buffer with *NO PADDING*
578 // Therefore ConvertF32toU8 function not used
James Wardc15f7d52022-12-07 15:38:01 +0000579 std::vector<uint8_t> u8_out;
James Ward80905bb2023-01-25 15:51:27 +0000580 uint32_t* val_u32 = reinterpret_cast<uint32_t*>(&f_in);
581 u8_out.push_back(*val_u32 & 0xFF);
582 u8_out.push_back((*val_u32 >> 8) & 0xFF);
583 u8_out.push_back((*val_u32 >> 16) & 0xFF);
584 u8_out.push_back((*val_u32 >> 24) & 0xFF);
James Wardc15f7d52022-12-07 15:38:01 +0000585 return u8_out;
586}
587
Kevin Chengb97cb1d2021-10-14 11:53:39 -0700588tosa_err_t TosaSerializationHandler::Serialize()
Eric Kunze2364dcd2021-04-26 11:06:57 -0700589{
Jerry Ge13c78a62022-10-04 20:32:39 -0700590 // regions
591 std::vector<flatbuffers::Offset<TosaRegion>> fboffset_regions;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700592
Eric Kunze2364dcd2021-04-26 11:06:57 -0700593 // translate TosaFlatbufferOperator to flatbuffers::Offset<TosaOperator>
Jerry Ge13c78a62022-10-04 20:32:39 -0700594 for (auto region : GetRegions())
Eric Kunze2364dcd2021-04-26 11:06:57 -0700595 {
Tai Lycfcb20d2023-03-13 21:04:11 +0000596 std::vector<flatbuffers::Offset<TosaBasicBlock>> fboffset_blocks;
Jerry Ge13c78a62022-10-04 20:32:39 -0700597 for (auto block : region->GetBlocks())
Eric Kunze2364dcd2021-04-26 11:06:57 -0700598 {
Tai Lycfcb20d2023-03-13 21:04:11 +0000599 std::vector<flatbuffers::Offset<TosaOperator>> fboffset_block_operators;
600 std::vector<flatbuffers::Offset<TosaTensor>> fboffset_block_tensors;
601 std::vector<flatbuffers::Offset<flatbuffers::String>> fboffset_block_inputs;
602 std::vector<flatbuffers::Offset<flatbuffers::String>> fboffset_block_outputs;
Jerry Ge13c78a62022-10-04 20:32:39 -0700603 auto block_name = _builder.CreateString(block->GetName().c_str());
604 for (auto tensor_str : block->GetInputs())
Eric Kunze2364dcd2021-04-26 11:06:57 -0700605 {
606 auto tensor_name = _builder.CreateString(tensor_str.c_str());
Jerry Ge13c78a62022-10-04 20:32:39 -0700607 fboffset_block_inputs.push_back(tensor_name);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700608 }
Jerry Ge13c78a62022-10-04 20:32:39 -0700609 for (auto tensor_str : block->GetOutputs())
Eric Kunze2364dcd2021-04-26 11:06:57 -0700610 {
611 auto tensor_name = _builder.CreateString(tensor_str.c_str());
Jerry Ge13c78a62022-10-04 20:32:39 -0700612 fboffset_block_outputs.push_back(tensor_name);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700613 }
Jerry Ge13c78a62022-10-04 20:32:39 -0700614 auto fb_block_inputs = _builder.CreateVector(fboffset_block_inputs);
615 auto fb_block_outputs = _builder.CreateVector(fboffset_block_outputs);
616 for (auto op : block->GetOperators())
Eric Kunze2364dcd2021-04-26 11:06:57 -0700617 {
Tai Lycfcb20d2023-03-13 21:04:11 +0000618 std::vector<flatbuffers::Offset<flatbuffers::String>> fboffset_operator_inputs;
619 std::vector<flatbuffers::Offset<flatbuffers::String>> fboffset_operator_outputs;
Jerry Ge13c78a62022-10-04 20:32:39 -0700620 auto operator_op = op->GetOp();
621 auto attribute_type = op->GetAttributeType();
622 for (auto tensor_str : op->GetInputTensorNames())
623 {
624 auto tensor_name = _builder.CreateString(tensor_str.c_str());
625 fboffset_operator_inputs.push_back(tensor_name);
626 }
627 for (auto tensor_str : op->GetOutputTensorNames())
628 {
629 auto tensor_name = _builder.CreateString(tensor_str.c_str());
630 fboffset_operator_outputs.push_back(tensor_name);
631 }
632 auto fb_operator_inputs = _builder.CreateVector(fboffset_operator_inputs);
633 auto fb_operator_outputs = _builder.CreateVector(fboffset_operator_outputs);
634 flatbuffers::Offset<void> fb_attribute;
635 switch (attribute_type)
636 {
637 case Attribute_NONE:
638 fb_attribute = 0;
639 break;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700640#define DEF_ARGS_S_STR(NAME, V) , _builder.CreateString(reinterpret_cast<Tosa##NAME*>(op->GetAttribute())->V().c_str())
James Wardc15f7d52022-12-07 15:38:01 +0000641#define DEF_ARGS_S_FP_as_U8(NAME, V) \
James Ward80905bb2023-01-25 15:51:27 +0000642 , _builder.CreateVector<uint8_t>(float_to_u8_helper(reinterpret_cast<Tosa##NAME*>(op->GetAttribute())->V()))
Eric Kunze2364dcd2021-04-26 11:06:57 -0700643#define DEF_ARGS_S_DEFAULT(NAME, V) , reinterpret_cast<Tosa##NAME*>(op->GetAttribute())->V()
Eric Kunze2364dcd2021-04-26 11:06:57 -0700644#define DEF_ARGS_S_int32_t(NAME, V) DEF_ARGS_S_DEFAULT(NAME, V)
James Wardc15f7d52022-12-07 15:38:01 +0000645#define DEF_ARGS_S_float(NAME, V) DEF_ARGS_S_FP_as_U8(NAME, V)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700646#define DEF_ARGS_S_bool(NAME, V) DEF_ARGS_S_DEFAULT(NAME, V)
647#define DEF_ARGS_S_ResizeMode(NAME, V) DEF_ARGS_S_DEFAULT(NAME, V)
James Ward485a11d2022-08-05 13:48:37 +0100648#define DEF_ARGS_S_DType(NAME, V) DEF_ARGS_S_DEFAULT(NAME, V)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700649#define DEF_ARGS_S_string(NAME, V) DEF_ARGS_S_STR(NAME, V)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700650#define DEF_ARGS_S(NAME, T, V) DEF_ARGS_S_##T(NAME, V)
651#define DEF_ARGS_V(NAME, T, V) , _builder.CreateVector<T>(reinterpret_cast<Tosa##NAME*>(op->GetAttribute())->V())
Eric Kunze2364dcd2021-04-26 11:06:57 -0700652#define DEF_ARGS_1(NAME, T0, F0, V0) DEF_ARGS_##F0(NAME, T0, V0)
653#define DEF_ARGS_2(NAME, T0, F0, V0, T1, F1, V1) DEF_ARGS_##F0(NAME, T0, V0) DEF_ARGS_##F1(NAME, T1, V1)
654#define DEF_ARGS_3(NAME, T0, F0, V0, T1, F1, V1, T2, F2, V2) \
655 DEF_ARGS_##F0(NAME, T0, V0) DEF_ARGS_##F1(NAME, T1, V1) DEF_ARGS_##F2(NAME, T2, V2)
656#define DEF_ARGS_4(NAME, T0, F0, V0, T1, F1, V1, T2, F2, V2, T3, F3, V3) \
657 DEF_ARGS_##F0(NAME, T0, V0) DEF_ARGS_##F1(NAME, T1, V1) DEF_ARGS_##F2(NAME, T2, V2) DEF_ARGS_##F3(NAME, T3, V3)
658#define DEF_ARGS_5(NAME, T0, F0, V0, T1, F1, V1, T2, F2, V2, T3, F3, V3, T4, F4, V4) \
659 DEF_ARGS_##F0(NAME, T0, V0) DEF_ARGS_##F1(NAME, T1, V1) DEF_ARGS_##F2(NAME, T2, V2) DEF_ARGS_##F3(NAME, T3, V3) \
660 DEF_ARGS_##F4(NAME, T4, V4)
661#define DEF_ARGS_6(NAME, T0, F0, V0, T1, F1, V1, T2, F2, V2, T3, F3, V3, T4, F4, V4, T5, F5, V5) \
662 DEF_ARGS_##F0(NAME, T0, V0) DEF_ARGS_##F1(NAME, T1, V1) DEF_ARGS_##F2(NAME, T2, V2) DEF_ARGS_##F3(NAME, T3, V3) \
663 DEF_ARGS_##F4(NAME, T4, V4) DEF_ARGS_##F5(NAME, T5, V5)
664#define DEF_ARGS_7(NAME, T0, F0, V0, T1, F1, V1, T2, F2, V2, T3, F3, V3, T4, F4, V4, T5, F5, V5, T6, F6, V6) \
665 DEF_ARGS_##F0(NAME, T0, V0) DEF_ARGS_##F1(NAME, T1, V1) DEF_ARGS_##F2(NAME, T2, V2) DEF_ARGS_##F3(NAME, T3, V3) \
666 DEF_ARGS_##F4(NAME, T4, V4) DEF_ARGS_##F5(NAME, T5, V5) DEF_ARGS_##F6(NAME, T6, V6)
Eric Kunze9601cbd2023-08-17 20:44:39 +0000667#define DEF_ARGS_8(NAME, T0, F0, V0, T1, F1, V1, T2, F2, V2, T3, F3, V3, T4, F4, V4, T5, F5, V5, T6, F6, V6, T7, F7, \
668 V7) \
669 DEF_ARGS_##F0(NAME, T0, V0) DEF_ARGS_##F1(NAME, T1, V1) DEF_ARGS_##F2(NAME, T2, V2) DEF_ARGS_##F3(NAME, T3, V3) \
670 DEF_ARGS_##F4(NAME, T4, V4) DEF_ARGS_##F5(NAME, T5, V5) DEF_ARGS_##F6(NAME, T6, V6) \
671 DEF_ARGS_##F7(NAME, T7, V7)
672#define DEF_ARGS_9(NAME, T0, F0, V0, T1, F1, V1, T2, F2, V2, T3, F3, V3, T4, F4, V4, T5, F5, V5, T6, F6, V6, T7, F7, \
673 V7, T8, F8, V8) \
674 DEF_ARGS_##F0(NAME, T0, V0) DEF_ARGS_##F1(NAME, T1, V1) DEF_ARGS_##F2(NAME, T2, V2) DEF_ARGS_##F3(NAME, T3, V3) \
675 DEF_ARGS_##F4(NAME, T4, V4) DEF_ARGS_##F5(NAME, T5, V5) DEF_ARGS_##F6(NAME, T6, V6) \
676 DEF_ARGS_##F7(NAME, T7, V7) DEF_ARGS_##F8(NAME, T8, V8)
Eric Kunze2364dcd2021-04-26 11:06:57 -0700677#define DEF_ATTRIBUTE(NAME, NUM_ARGS, ...) \
678 case Attribute_##NAME##Attribute: \
679 fb_attribute = Create##NAME##Attribute(_builder DEF_ARGS_##NUM_ARGS(NAME##Attribute, __VA_ARGS__)).Union(); \
680 break;
Eric Kunze2364dcd2021-04-26 11:06:57 -0700681#include "attribute.def"
682#undef DEF_ATTRIBUTE
683#undef DEF_ARGS_1
684#undef DEF_ARGS_2
685#undef DEF_ARGS_3
686#undef DEF_ARGS_4
687#undef DEF_ARGS_5
688#undef DEF_ARGS_6
689#undef DEF_ARGS_7
690#undef DEF_ARGS_S
691#undef DEF_ARGS_V
692#undef DEF_ARGS_S_int32_t
693#undef DEF_ARGS_S_float
694#undef DEF_ARGS_S_bool
695#undef DEF_ARGS_S_ResizeMode
James Ward485a11d2022-08-05 13:48:37 +0100696#undef DEF_ARGS_S_DType
Eric Kunze2364dcd2021-04-26 11:06:57 -0700697#undef DEF_ARGS_S_string
698#undef DEF_ARGS_S_STR
699#undef DEF_ARGS_S_DEFAULT
Jerry Ge13c78a62022-10-04 20:32:39 -0700700 default:
701 printf("TosaSerializationHandler::Serialize(): Attribute %s not implemented yet\n",
702 EnumNamesAttribute()[attribute_type]);
703 return TOSA_INTERNAL_ERROR;
704 }
705 auto fboffset_operator = CreateTosaOperator(_builder, operator_op, attribute_type, fb_attribute,
706 fb_operator_inputs, fb_operator_outputs);
707 fboffset_block_operators.push_back(fboffset_operator);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700708 }
Jerry Ge13c78a62022-10-04 20:32:39 -0700709 auto fb_block_operators = _builder.CreateVector(fboffset_block_operators);
710 for (auto tensor : block->GetTensors())
711 {
Tai Lyd0520b92023-09-19 21:30:18 +0000712 auto tensor_name = _builder.CreateString(tensor->GetName().c_str());
713 auto tensor_shape = _builder.CreateVector(tensor->GetShape());
714 auto tensor_dtype = tensor->GetDtype();
715 bool tensor_variable = tensor->GetVariable();
716 auto tensor_data = _builder.CreateVector(tensor->GetData());
717 auto tensor_is_unranked = tensor->GetIsUnranked();
718 auto tensor_variable_name = _builder.CreateString(tensor->GetVariableName().c_str());
Tai Lyc6939a42023-08-21 17:00:29 +0000719 auto fboffset_tensor = CreateTosaTensor(_builder, tensor_name, tensor_shape, tensor_dtype, tensor_data,
Tai Lyd0520b92023-09-19 21:30:18 +0000720 tensor_variable, tensor_is_unranked, tensor_variable_name);
Jerry Ge13c78a62022-10-04 20:32:39 -0700721 fboffset_block_tensors.push_back(fboffset_tensor);
722 }
723 auto fb_block_tensors = _builder.CreateVector(fboffset_block_tensors);
724 auto fboffset_block = CreateTosaBasicBlock(_builder, block_name, fb_block_operators, fb_block_tensors,
Won Jeoncb4bbf42023-08-10 08:50:15 +0000725 fb_block_inputs, fb_block_outputs);
Jerry Ge13c78a62022-10-04 20:32:39 -0700726 fboffset_blocks.push_back(fboffset_block);
727 } // end block for_loop
728 auto fb_blocks = _builder.CreateVector(fboffset_blocks);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700729
Jerry Ge13c78a62022-10-04 20:32:39 -0700730 auto region_name = _builder.CreateString(region->GetName().c_str());
731 auto fboffset_region = CreateTosaRegion(_builder, region_name, fb_blocks);
732 fboffset_regions.push_back(fboffset_region);
733 } // end region for_loop
Eric Kunze2364dcd2021-04-26 11:06:57 -0700734
Jerry Ge13c78a62022-10-04 20:32:39 -0700735 auto fb_regions = _builder.CreateVector(fboffset_regions);
Eric Kunze2364dcd2021-04-26 11:06:57 -0700736
Kevin Chengb97cb1d2021-10-14 11:53:39 -0700737 auto fb_version =
738 CreateVersion(_builder, TOSA_VERSION_MAJOR, TOSA_VERSION_MINOR, TOSA_VERSION_PATCH, TOSA_VERSION_DRAFT);
Jerry Ge13c78a62022-10-04 20:32:39 -0700739 auto fb_graph = CreateTosaGraph(_builder, fb_version, fb_regions);
Eric Kunzee6596402022-06-09 21:27:36 +0000740 _builder.Finish(fb_graph, TosaGraphIdentifier());
Eric Kunze2364dcd2021-04-26 11:06:57 -0700741
742 return TOSA_OK;
743}
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700744
Jerry Ge442261b2022-09-09 13:38:56 -0700745void TosaSerializationHandler::ForceAlignTensorData(std::vector<uint8_t>& buf)
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700746{
747 while ((buf.size() % TENSOR_BUFFER_FORCE_ALIGNMENT) != 0)
748 {
749 buf.push_back(0);
750 }
751}
752
Tai Lyce911a22024-03-21 17:01:14 +0000753tosa_err_t TosaSerializationHandler::ConvertBF16toU8(const std::vector<float>& in, std::vector<uint8_t>& out)
754{
755 // Note: Converts fp32->bf16 by ignoring the least significant 16 bits
756 out.clear();
757 for (auto val : in)
758 {
759 uint32_t* val_u32 = reinterpret_cast<uint32_t*>(&val);
760 uint8_t f32_byte2 = (*val_u32 >> 16) & 0xFF;
761 uint8_t f32_byte3 = (*val_u32 >> 24) & 0xFF;
762 // little endian: byte2 followed by byte3
763 out.push_back(f32_byte2);
764 out.push_back(f32_byte3);
765 }
766 ForceAlignTensorData(out);
767 return TOSA_OK;
768}
769
770tosa_err_t TosaSerializationHandler::ConvertFP8E4M3toU8(const std::vector<float>& in, std::vector<uint8_t>& out)
771{
772 // Note: Converts fp32->FP8E4M3 before converting to unint8_t
773 out.clear();
774 for (auto val : in)
775 {
776 auto f8 = static_cast<fp8e4m3>(val);
777 uint8_t b8 = f8.bits();
778 out.push_back(b8);
779 }
780 ForceAlignTensorData(out);
781 return TOSA_OK;
782}
783
784tosa_err_t TosaSerializationHandler::ConvertFP8E5M2toU8(const std::vector<float>& in, std::vector<uint8_t>& out)
785{
786 // Note: Converts fp32->FP8E5M2 before converting to uint8_t
787 out.clear();
788 for (auto val : in)
789 {
790 auto f8 = static_cast<fp8e5m2>(val);
791 uint8_t b8 = f8.bits();
792 out.push_back(b8);
793 }
794 ForceAlignTensorData(out);
795 return TOSA_OK;
796}
797
James Ward485a11d2022-08-05 13:48:37 +0100798tosa_err_t TosaSerializationHandler::ConvertF16toU8(const std::vector<float>& in, std::vector<uint8_t>& out)
799{
800 // Note: Converts fp32->fp16 before converting to uint8_t
801 out.clear();
802 for (auto val : in)
803 {
804 half_float::half val_f16 = half_float::half_cast<half_float::half, float>(val);
805 uint16_t* val_u16 = reinterpret_cast<uint16_t*>(&val_f16);
806 out.push_back(*val_u16 & 0xFF);
807 out.push_back((*val_u16 >> 8) & 0xFF);
808 }
Jerry Ge442261b2022-09-09 13:38:56 -0700809 ForceAlignTensorData(out);
James Ward485a11d2022-08-05 13:48:37 +0100810 return TOSA_OK;
811}
812
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700813tosa_err_t TosaSerializationHandler::ConvertF32toU8(const std::vector<float>& in, std::vector<uint8_t>& out)
814{
815 out.clear();
816 for (auto val : in)
817 {
818 uint32_t* val_u32 = reinterpret_cast<uint32_t*>(&val);
819 out.push_back(*val_u32 & 0xFF);
820 out.push_back((*val_u32 >> 8) & 0xFF);
821 out.push_back((*val_u32 >> 16) & 0xFF);
822 out.push_back((*val_u32 >> 24) & 0xFF);
823 }
Jerry Ge442261b2022-09-09 13:38:56 -0700824 ForceAlignTensorData(out);
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700825 return TOSA_OK;
826}
827
Tai Ly5d580fa2023-12-15 20:34:51 +0000828tosa_err_t TosaSerializationHandler::ConvertI64toU8(const std::vector<int64_t>& in, std::vector<uint8_t>& out)
829{
830 out.clear();
831 for (auto val : in)
832 {
833 uint64_t* val_u64 = reinterpret_cast<uint64_t*>(&val);
834 out.push_back(*val_u64 & 0xFF);
835 out.push_back((*val_u64 >> 8) & 0xFF);
836 out.push_back((*val_u64 >> 16) & 0xFF);
837 out.push_back((*val_u64 >> 24) & 0xFF);
838 out.push_back((*val_u64 >> 32) & 0xFF);
839 out.push_back((*val_u64 >> 40) & 0xFF);
840 out.push_back((*val_u64 >> 48) & 0xFF);
841 out.push_back((*val_u64 >> 56) & 0xFF);
842 }
843 ForceAlignTensorData(out);
844 return TOSA_OK;
845}
846
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700847tosa_err_t TosaSerializationHandler::ConvertI48toU8(const std::vector<int64_t>& in, std::vector<uint8_t>& out)
848{
849 out.clear();
850 for (auto val : in)
851 {
852 uint64_t* val_u64 = reinterpret_cast<uint64_t*>(&val);
853 out.push_back(*val_u64 & 0xFF);
854 out.push_back((*val_u64 >> 8) & 0xFF);
855 out.push_back((*val_u64 >> 16) & 0xFF);
856 out.push_back((*val_u64 >> 24) & 0xFF);
857 out.push_back((*val_u64 >> 32) & 0xFF);
858 out.push_back((*val_u64 >> 40) & 0xFF);
859 }
Jerry Ge442261b2022-09-09 13:38:56 -0700860 ForceAlignTensorData(out);
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700861 return TOSA_OK;
862}
863
864tosa_err_t TosaSerializationHandler::ConvertI32toU8(const std::vector<int32_t>& in, std::vector<uint8_t>& out)
865{
866 out.clear();
867 for (auto val : in)
868 {
869 uint32_t* val_u32 = reinterpret_cast<uint32_t*>(&val);
870 out.push_back(*val_u32 & 0xFF);
871 out.push_back((*val_u32 >> 8) & 0xFF);
872 out.push_back((*val_u32 >> 16) & 0xFF);
873 out.push_back((*val_u32 >> 24) & 0xFF);
874 }
Jerry Ge442261b2022-09-09 13:38:56 -0700875 ForceAlignTensorData(out);
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700876 return TOSA_OK;
877}
878
879tosa_err_t TosaSerializationHandler::ConvertI16toU8(const std::vector<int16_t>& in, std::vector<uint8_t>& out)
880{
881 out.clear();
882 for (auto val : in)
883 {
884 uint16_t* val_u16 = reinterpret_cast<uint16_t*>(&val);
885 out.push_back(*val_u16 & 0xFF);
886 out.push_back((*val_u16 >> 8) & 0xFF);
887 }
Jerry Ge442261b2022-09-09 13:38:56 -0700888 ForceAlignTensorData(out);
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700889 return TOSA_OK;
890}
891
892tosa_err_t TosaSerializationHandler::ConvertI8toU8(const std::vector<int8_t>& in, std::vector<uint8_t>& out)
893{
894 out.clear();
895 for (auto val : in)
896 {
897 uint8_t* val_u8 = reinterpret_cast<uint8_t*>(&val);
898 out.push_back(*val_u8);
899 }
Jerry Ge442261b2022-09-09 13:38:56 -0700900 ForceAlignTensorData(out);
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700901 return TOSA_OK;
902}
903
Kevin Cheng3ce56342021-07-28 13:42:29 -0700904// Two int4 values are packed into one byte out.
905// For given input value val_0 = in[2*i], and val_1 = in[2*i+1],
906// they'll be packed as out[3:0] = val_0, and out[7:4] = val_1
907tosa_err_t TosaSerializationHandler::ConvertI4toU8(const std::vector<int8_t>& in, std::vector<uint8_t>& out)
908{
909 out.clear();
910 uint32_t in_size = in.size();
911 uint32_t out_size = (in_size % 2 == 0) ? (in_size / 2) : ((in_size + 1) / 2);
Eric Kunzeb13fe8f2022-02-17 17:14:25 -0800912 for (uint32_t i = 0; i < out_size; i++)
Kevin Cheng3ce56342021-07-28 13:42:29 -0700913 {
914 int8_t val_0 = in[2 * i];
915 int8_t val_1 = 0;
Eric Kunzeb13fe8f2022-02-17 17:14:25 -0800916 if (2u * i + 1u < in_size)
Kevin Cheng3ce56342021-07-28 13:42:29 -0700917 {
918 val_1 = in[2 * i + 1];
919 }
920 // In TOSA spec, int4 ranges [-7, 7]
921 if (val_0 < -7 || val_0 > 7 || val_1 < -7 || val_1 > 7)
922 {
923 printf("TosaSerializationHandler::ConvertI4toU8(): element in input array (%d or %d) exceeds int4 range.\n",
924 val_0, val_1);
925 return TOSA_USER_ERROR;
926 }
927 int8_t val_packed = (val_0 & 0xF) | ((val_1 & 0xF) << 4);
928 uint8_t val_u8 = static_cast<uint8_t>(val_packed);
929 out.push_back(val_u8);
930 }
Jerry Ge442261b2022-09-09 13:38:56 -0700931 ForceAlignTensorData(out);
Kevin Cheng3ce56342021-07-28 13:42:29 -0700932 return TOSA_OK;
933}
934
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700935tosa_err_t TosaSerializationHandler::ConvertBooltoU8(const std::vector<bool>& in, std::vector<uint8_t>& out)
936{
937 out.clear();
938 for (auto val : in)
939 {
Eric Kunze4417b422022-06-20 07:27:42 -0700940 uint8_t val_u8 = val;
941 out.push_back(val_u8);
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700942 }
Jerry Ge442261b2022-09-09 13:38:56 -0700943 ForceAlignTensorData(out);
Kevin Cheng3bb1bc12021-06-17 15:57:08 -0700944 return TOSA_OK;
945}
946
Tai Lyce911a22024-03-21 17:01:14 +0000947tosa_err_t TosaSerializationHandler::ConvertU8toBF16(const std::vector<uint8_t>& in,
948 uint32_t out_size,
949 std::vector<float>& out)
950{
951 // Note: bf16 values returned in fp32 type
952 out.clear();
953 if (in.size() < out_size * sizeof(int16_t))
954 {
955 printf("TosaSerializationHandler::ConvertU8toBF16(): uint8 buffer size %ld must >= target size %ld\n",
956 in.size(), out_size * sizeof(int16_t));
957 return TOSA_USER_ERROR;
958 }
959
960 for (uint32_t i = 0; i < out_size; i++)
961 {
962 uint32_t f32_byte2 = in[i * sizeof(int16_t)];
963 uint32_t f32_byte3 = in[i * sizeof(int16_t) + 1];
964 uint32_t val_u32 = (f32_byte2 << 16) + (f32_byte3 << 24);
965
966 // Reinterpret u32 bytes as fp32
967 float val_f32 = *(float*)&val_u32;
968 out.push_back(val_f32);
969 }
970 return TOSA_OK;
971}
972
973tosa_err_t TosaSerializationHandler::ConvertU8toFP8E4M3(const std::vector<uint8_t>& in,
974 uint32_t out_size,
975 std::vector<float>& out)
976{
977 // Note: FP8E4M3 values returned in fp32 type
978 out.clear();
979 if (in.size() < out_size * sizeof(int8_t))
980 {
981 printf("TosaSerializationHandler::ConvertU8toF16(): uint8 buffer size %ld must >= target size %ld\n", in.size(),
982 out_size * sizeof(int8_t));
983 return TOSA_USER_ERROR;
984 }
985
986 for (uint32_t i = 0; i < out_size; i++)
987 {
988 int8_t bits = static_cast<int8_t>(in[i * sizeof(int8_t)]);
989 auto f8 = fp8e4m3::from_bits(bits);
990 float val_f32 = static_cast<float>(f8);
991 out.push_back(val_f32);
992 }
993 return TOSA_OK;
994}
995
996tosa_err_t TosaSerializationHandler::ConvertU8toFP8E5M2(const std::vector<uint8_t>& in,
997 uint32_t out_size,
998 std::vector<float>& out)
999{
1000 // Note: FP8E5M2 values returned in fp32 type
1001 out.clear();
1002 if (in.size() < out_size * sizeof(int8_t))
1003 {
1004 printf("TosaSerializationHandler::ConvertU8toF16(): uint8 buffer size %ld must >= target size %ld\n", in.size(),
1005 out_size * sizeof(int8_t));
1006 return TOSA_USER_ERROR;
1007 }
1008
1009 for (uint32_t i = 0; i < out_size; i++)
1010 {
1011 int8_t bits = static_cast<int8_t>(in[i * sizeof(int8_t)]);
1012 auto f8 = fp8e5m2::from_bits(bits);
1013 float val_f32 = static_cast<float>(f8);
1014 out.push_back(val_f32);
1015 }
1016 return TOSA_OK;
1017}
1018
Jerry Ge758e73e2024-02-26 13:31:22 -08001019tosa_err_t TosaSerializationHandler::ConvertU8toF16(const std::vector<uint8_t>& in,
1020 uint32_t out_size,
1021 std::vector<half_float::half>& out)
James Ward485a11d2022-08-05 13:48:37 +01001022{
1023 // Note: fp16 values returned in fp32 type
1024 out.clear();
1025 if (in.size() < out_size * sizeof(int16_t))
1026 {
1027 printf("TosaSerializationHandler::ConvertU8toF16(): uint8 buffer size %ld must >= target size %ld\n", in.size(),
1028 out_size * sizeof(int16_t));
1029 return TOSA_USER_ERROR;
1030 }
1031
1032 for (uint32_t i = 0; i < out_size; i++)
1033 {
1034 uint16_t f16_byte0 = in[i * sizeof(int16_t)];
1035 uint16_t f16_byte1 = in[i * sizeof(int16_t) + 1];
1036 uint16_t val_u16 = f16_byte0 + (f16_byte1 << 8);
1037
1038 // Reinterpret u16 byte as fp16 then convert to fp32
1039 half_float::half val_f16 = *(half_float::half*)&val_u16;
Jerry Ge758e73e2024-02-26 13:31:22 -08001040 out.push_back(val_f16);
James Ward485a11d2022-08-05 13:48:37 +01001041 }
1042 return TOSA_OK;
1043}
1044
1045tosa_err_t
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001046 TosaSerializationHandler::ConvertU8toF32(const std::vector<uint8_t>& in, uint32_t out_size, std::vector<float>& out)
1047{
1048 out.clear();
1049 if (in.size() < out_size * sizeof(float))
1050 {
1051 printf("TosaSerializationHandler::ConvertU8toF32(): uint8 buffer size %ld must >= target size %ld\n", in.size(),
1052 out_size * sizeof(float));
1053 return TOSA_USER_ERROR;
1054 }
Eric Kunzeb13fe8f2022-02-17 17:14:25 -08001055 for (uint32_t i = 0; i < out_size; i++)
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001056 {
1057 uint32_t byte0 = in[i * sizeof(float)];
1058 uint32_t byte1 = in[i * sizeof(float) + 1];
1059 uint32_t byte2 = in[i * sizeof(float) + 2];
1060 uint32_t byte3 = in[i * sizeof(float) + 3];
1061 uint32_t val_u32 = byte0 + (byte1 << 8) + (byte2 << 16) + (byte3 << 24);
1062 float* val_fp32 = reinterpret_cast<float*>(&val_u32);
1063 out.push_back(*val_fp32);
1064 }
1065 return TOSA_OK;
1066}
1067
Tai Ly5d580fa2023-12-15 20:34:51 +00001068tosa_err_t TosaSerializationHandler::ConvertU8toI64(const std::vector<uint8_t>& in,
1069 uint32_t out_size,
1070 std::vector<int64_t>& out)
1071{
1072 out.clear();
1073 if (in.size() < out_size * sizeof(int64_t))
1074 {
1075 printf("TosaSerializationHandler::ConvertU8toI64(): uint8 buffer size %ld must >= target size %ld\n", in.size(),
1076 out_size * sizeof(int64_t));
1077 return TOSA_USER_ERROR;
1078 }
1079 for (uint32_t i = 0; i < out_size; i++)
1080 {
1081 uint64_t byte0 = in[i * sizeof(int64_t)];
1082 uint64_t byte1 = in[i * sizeof(int64_t) + 1];
1083 uint64_t byte2 = in[i * sizeof(int64_t) + 2];
1084 uint64_t byte3 = in[i * sizeof(int64_t) + 3];
1085 uint64_t byte4 = in[i * sizeof(int64_t) + 4];
1086 uint64_t byte5 = in[i * sizeof(int64_t) + 5];
1087 uint64_t byte6 = in[i * sizeof(int64_t) + 6];
1088 uint64_t byte7 = in[i * sizeof(int64_t) + 7];
1089 uint64_t val_u64 = byte0 + (byte1 << 8) + (byte2 << 16) + (byte3 << 24) + (byte4 << 32) + (byte5 << 40) +
1090 (byte6 << 48) + (byte7 << 56);
1091 int64_t* val_i64 = reinterpret_cast<int64_t*>(&val_u64);
1092 out.push_back(*val_i64);
1093 }
1094 return TOSA_OK;
1095}
1096
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001097tosa_err_t TosaSerializationHandler::ConvertU8toI48(const std::vector<uint8_t>& in,
1098 uint32_t out_size,
1099 std::vector<int64_t>& out)
1100{
1101 out.clear();
1102 if (in.size() < out_size * 6 /* sizeof(int48) */)
1103 {
1104 printf("TosaSerializationHandler::ConvertU8toI48(): uint8 buffer size %ld must >= target size %d\n", in.size(),
1105 out_size * 6);
1106 return TOSA_USER_ERROR;
1107 }
Eric Kunzeb13fe8f2022-02-17 17:14:25 -08001108 for (uint32_t i = 0; i < out_size; i++)
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001109 {
1110 uint64_t byte0 = in[i * 6];
1111 uint64_t byte1 = in[i * 6 + 1];
1112 uint64_t byte2 = in[i * 6 + 2];
1113 uint64_t byte3 = in[i * 6 + 3];
1114 uint64_t byte4 = in[i * 6 + 4];
1115 uint64_t byte5 = in[i * 6 + 5];
1116 bool sign = ((byte5 >> 7) & 1) == 1 ? true : false;
1117 uint64_t val_u64 = byte0 + (byte1 << 8) + (byte2 << 16) + (byte3 << 24) + (byte4 << 32) + (byte5 << 40);
1118 if (sign)
1119 {
1120 uint64_t sext_mask = (0xFFFFUL << 48);
1121 val_u64 |= sext_mask;
1122 }
1123 int64_t* val_i64 = reinterpret_cast<int64_t*>(&val_u64);
1124 out.push_back(*val_i64);
1125 }
1126 return TOSA_OK;
1127}
1128
1129tosa_err_t TosaSerializationHandler::ConvertU8toI32(const std::vector<uint8_t>& in,
1130 uint32_t out_size,
1131 std::vector<int32_t>& out)
1132{
1133 out.clear();
1134 if (in.size() < out_size * sizeof(int32_t))
1135 {
1136 printf("TosaSerializationHandler::ConvertU8toI32(): uint8 buffer size %ld must >= target size %ld\n", in.size(),
1137 out_size * sizeof(int32_t));
1138 return TOSA_USER_ERROR;
1139 }
Eric Kunzeb13fe8f2022-02-17 17:14:25 -08001140 for (uint32_t i = 0; i < out_size; i++)
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001141 {
1142 uint32_t byte0 = in[i * sizeof(int32_t)];
1143 uint32_t byte1 = in[i * sizeof(int32_t) + 1];
1144 uint32_t byte2 = in[i * sizeof(int32_t) + 2];
1145 uint32_t byte3 = in[i * sizeof(int32_t) + 3];
1146 uint32_t val_u32 = byte0 + (byte1 << 8) + (byte2 << 16) + (byte3 << 24);
1147 int32_t* val_i32 = reinterpret_cast<int32_t*>(&val_u32);
1148 out.push_back(*val_i32);
1149 }
1150 return TOSA_OK;
1151}
1152
1153tosa_err_t TosaSerializationHandler::ConvertU8toI16(const std::vector<uint8_t>& in,
1154 uint32_t out_size,
1155 std::vector<int16_t>& out)
1156{
1157 out.clear();
1158 if (in.size() < out_size * sizeof(int16_t))
1159 {
1160 printf("TosaSerializationHandler::ConvertU8toI16(): uint8 buffer size %ld must >= target size %ld\n", in.size(),
1161 out_size * sizeof(int16_t));
1162 return TOSA_USER_ERROR;
1163 }
Eric Kunzeb13fe8f2022-02-17 17:14:25 -08001164 for (uint32_t i = 0; i < out_size; i++)
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001165 {
1166 uint16_t byte0 = in[i * sizeof(int16_t)];
1167 uint16_t byte1 = in[i * sizeof(int16_t) + 1];
1168 uint16_t val_u16 = byte0 + (byte1 << 8);
1169 int16_t* val_i16 = reinterpret_cast<int16_t*>(&val_u16);
1170 out.push_back(*val_i16);
1171 }
1172 return TOSA_OK;
1173}
1174
1175tosa_err_t
1176 TosaSerializationHandler::ConvertU8toI8(const std::vector<uint8_t>& in, uint32_t out_size, std::vector<int8_t>& out)
1177{
1178 out.clear();
1179 if (in.size() < out_size * sizeof(int8_t))
1180 {
1181 printf("TosaSerializationHandler::ConvertU8toI8(): uint8 buffer size %ld must >= target size %ld\n", in.size(),
Kevin Cheng3ce56342021-07-28 13:42:29 -07001182 out_size * sizeof(int8_t));
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001183 return TOSA_USER_ERROR;
1184 }
Eric Kunzeb13fe8f2022-02-17 17:14:25 -08001185 for (uint32_t i = 0; i < out_size; i++)
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001186 {
1187 uint8_t val_u8 = in[i];
1188 int8_t* val_i8 = reinterpret_cast<int8_t*>(&val_u8);
1189 out.push_back(*val_i8);
1190 }
1191 return TOSA_OK;
1192}
1193
1194tosa_err_t
Kevin Cheng3ce56342021-07-28 13:42:29 -07001195 TosaSerializationHandler::ConvertU8toI4(const std::vector<uint8_t>& in, uint32_t out_size, std::vector<int8_t>& out)
1196{
1197 out.clear();
1198 if (out_size > in.size() * 2)
1199 {
1200 printf("TosaSerializationHandler::ConvertU8toI4(): output size %u must <= uint8 buffer size %ld x 2.\n",
1201 out_size, in.size());
1202 return TOSA_USER_ERROR;
1203 }
Eric Kunzeb13fe8f2022-02-17 17:14:25 -08001204 for (size_t i = 0; i < in.size(); i++)
Kevin Cheng3ce56342021-07-28 13:42:29 -07001205 {
1206 uint8_t val_u8 = in[i];
1207 uint8_t val_0_u4 = val_u8 & 0xF;
1208 uint8_t val_1_u4 = val_u8 >> 4;
1209 uint8_t val_0_u8_sext = (val_0_u4 & 0x08) ? (val_0_u4 | 0xF0) : val_0_u4;
1210 uint8_t val_1_u8_sext = (val_1_u4 & 0x08) ? (val_1_u4 | 0xF0) : val_1_u4;
1211 int8_t val_0 = static_cast<int8_t>(val_0_u8_sext);
1212 int8_t val_1 = static_cast<int8_t>(val_1_u8_sext);
1213 // In TOSA spec, int4 ranges [-7, 7]
1214 if (val_0 < -7 || val_0 > 7 || val_1 < -7 || val_1 > 7)
1215 {
1216 printf(
1217 "TosaSerializationHandler::ConvertU8toI4(): element in output array (%d or %d) exceeds int4 range.\n",
1218 val_0, val_1);
1219 return TOSA_USER_ERROR;
1220 }
1221 out.push_back(val_0);
1222 if (2 * i + 1 < out_size)
1223 out.push_back(val_1);
1224 }
1225 return TOSA_OK;
1226}
1227
1228tosa_err_t
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001229 TosaSerializationHandler::ConvertU8toBool(const std::vector<uint8_t>& in, uint32_t out_size, std::vector<bool>& out)
1230{
1231 out.clear();
1232 if (in.size() < out_size * sizeof(bool))
1233 {
1234 printf("TosaSerializationHandler::ConvertU8toBool(): uint8 buffer size %ld must >= target size %ld\n",
1235 in.size(), out_size * sizeof(bool));
1236 return TOSA_USER_ERROR;
1237 }
Eric Kunzeb13fe8f2022-02-17 17:14:25 -08001238 for (uint32_t i = 0; i < out_size; i++)
Kevin Cheng3bb1bc12021-06-17 15:57:08 -07001239 {
1240 uint8_t val_u8 = in[i];
1241 bool* val_bool = reinterpret_cast<bool*>(&val_u8);
1242 out.push_back(*val_bool);
1243 }
1244 return TOSA_OK;
1245}