Gitiles
Code Review Sign In
review.mlplatform.org / ml / armnn / 75b981dba849db74b1cb74438348cbcad54bf45d / . / src / armnnSerializer / test / LstmSerializationTests.cpp
blob: bdc37877f76891ed5b790cf9da7d8d4980241ad3 [file] [log] [blame]
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]1//
2// Copyright © 2021 Arm Ltd and Contributors. All rights reserved.
3// SPDX-License-Identifier: MIT
4//
5
6#include "../Serializer.hpp"
7#include "SerializerTestUtils.hpp"
8
9#include <armnn/Descriptors.hpp>
10#include <armnn/INetwork.hpp>
11#include <armnn/IRuntime.hpp>
12#include <armnnDeserializer/IDeserializer.hpp>
13#include <armnn/utility/IgnoreUnused.hpp>
14#include <armnn/LstmParams.hpp>
15#include <armnn/QuantizedLstmParams.hpp>
16
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]17#include <doctest/doctest.h>
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]18#include <fmt/format.h>
19
20
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]21TEST_SUITE("SerializerTests")
22{
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]23template<typename Descriptor>
24armnn::LstmInputParams ConstantVector2LstmInputParams(const std::vector<armnn::ConstTensor>& constants,
25 Descriptor& descriptor)
26{
27 armnn::LstmInputParams lstmInputParams;
28 size_t i = 0;
29
30 // Inserting basic paramters
31 lstmInputParams.m_InputToForgetWeights = &constants[i++];
32 lstmInputParams.m_InputToCellWeights = &constants[i++];
33 lstmInputParams.m_InputToOutputWeights = &constants[i++];
34 lstmInputParams.m_RecurrentToForgetWeights = &constants[i++];
35 lstmInputParams.m_RecurrentToCellWeights = &constants[i++];
36 lstmInputParams.m_RecurrentToOutputWeights = &constants[i++];
37 lstmInputParams.m_ForgetGateBias = &constants[i++];
38 lstmInputParams.m_CellBias = &constants[i++];
39 lstmInputParams.m_OutputGateBias = &constants[i++];
40 if (!descriptor.m_CifgEnabled)
41 {
42 lstmInputParams.m_InputToInputWeights = &constants[i++];
43 lstmInputParams.m_RecurrentToInputWeights = &constants[i++];
44 lstmInputParams.m_InputGateBias = &constants[i++];
45 }
46
47 if (descriptor.m_PeepholeEnabled)
48 {
49 if (!descriptor.m_CifgEnabled)
50 {
51 lstmInputParams.m_CellToInputWeights = &constants[i++];
52 }
53 lstmInputParams.m_CellToForgetWeights = &constants[i++];
54 lstmInputParams.m_CellToOutputWeights = &constants[i++];
55 }
56
57 if (descriptor.m_ProjectionEnabled)
58 {
59 lstmInputParams.m_ProjectionWeights = &constants[i++];
60 lstmInputParams.m_ProjectionBias = &constants[i++];
61 }
62
63 if (descriptor.m_LayerNormEnabled)
64 {
65 if (!descriptor.m_CifgEnabled)
66 {
67 lstmInputParams.m_InputLayerNormWeights = &constants[i++];
68 }
69 lstmInputParams.m_ForgetLayerNormWeights = &constants[i++];
70 lstmInputParams.m_CellLayerNormWeights = &constants[i++];
71 lstmInputParams.m_OutputLayerNormWeights = &constants[i++];
72 }
73
74 return lstmInputParams;
75}
76
Narumol Prangnawarata0162e12021-07-23 14:47:49 +0100[diff] [blame]77// Works for Lstm, UnidirectionalSequenceLstm and QLstm (QuantizedLstm uses different parameters)
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]78template<typename Descriptor>
79class VerifyLstmLayer : public LayerVerifierBaseWithDescriptor<Descriptor>
80{
81public:
82 VerifyLstmLayer(const std::string& layerName,
83 const std::vector<armnn::TensorInfo>& inputInfos,
84 const std::vector<armnn::TensorInfo>& outputInfos,
85 const Descriptor& descriptor,
86 const armnn::LstmInputParams& inputParams)
87 : LayerVerifierBaseWithDescriptor<Descriptor>(layerName, inputInfos, outputInfos, descriptor)
88 , m_InputParams(inputParams) {}
89
90 void ExecuteStrategy(const armnn::IConnectableLayer* layer,
91 const armnn::BaseDescriptor& descriptor,
92 const std::vector<armnn::ConstTensor>& constants,
93 const char* name,
94 const armnn::LayerBindingId id = 0) override
95 {
96 armnn::IgnoreUnused(constants, id);
97 switch (layer->GetType())
98 {
99 case armnn::LayerType::Input: break;
100 case armnn::LayerType::Output: break;
101 case armnn::LayerType::Lstm:
Narumol Prangnawarata0162e12021-07-23 14:47:49 +0100[diff] [blame]102 case armnn::LayerType::UnidirectionalSequenceLstm:
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]103 {
104 this->VerifyNameAndConnections(layer, name);
105 const Descriptor& internalDescriptor = static_cast<const Descriptor&>(descriptor);
106 this->VerifyDescriptor(internalDescriptor);
107 armnn::LstmInputParams lstmParams = ConstantVector2LstmInputParams(constants, internalDescriptor);
108 VerifyInputParameters(lstmParams);
109 break;
110 }
111 case armnn::LayerType::QLstm:
112 {
113 this->VerifyNameAndConnections(layer, name);
114 const Descriptor& internalDescriptor = static_cast<const Descriptor&>(descriptor);
115 this->VerifyDescriptor(internalDescriptor);
116 armnn::LstmInputParams lstmParams = ConstantVector2LstmInputParams(constants, internalDescriptor);
117 VerifyInputParameters(lstmParams);
118 break;
119 }
120 default:
121 {
122 throw armnn::Exception("Unexpected layer type in Lstm test model");
123 }
124 }
125 }
126
127protected:
128 void VerifyInputParameters(const armnn::LstmInputParams& params)
129 {
130 this->VerifyConstTensors(
131 "m_InputToInputWeights", m_InputParams.m_InputToInputWeights, params.m_InputToInputWeights);
132 this->VerifyConstTensors(
133 "m_InputToForgetWeights", m_InputParams.m_InputToForgetWeights, params.m_InputToForgetWeights);
134 this->VerifyConstTensors(
135 "m_InputToCellWeights", m_InputParams.m_InputToCellWeights, params.m_InputToCellWeights);
136 this->VerifyConstTensors(
137 "m_InputToOutputWeights", m_InputParams.m_InputToOutputWeights, params.m_InputToOutputWeights);
138 this->VerifyConstTensors(
139 "m_RecurrentToInputWeights", m_InputParams.m_RecurrentToInputWeights, params.m_RecurrentToInputWeights);
140 this->VerifyConstTensors(
141 "m_RecurrentToForgetWeights", m_InputParams.m_RecurrentToForgetWeights, params.m_RecurrentToForgetWeights);
142 this->VerifyConstTensors(
143 "m_RecurrentToCellWeights", m_InputParams.m_RecurrentToCellWeights, params.m_RecurrentToCellWeights);
144 this->VerifyConstTensors(
145 "m_RecurrentToOutputWeights", m_InputParams.m_RecurrentToOutputWeights, params.m_RecurrentToOutputWeights);
146 this->VerifyConstTensors(
147 "m_CellToInputWeights", m_InputParams.m_CellToInputWeights, params.m_CellToInputWeights);
148 this->VerifyConstTensors(
149 "m_CellToForgetWeights", m_InputParams.m_CellToForgetWeights, params.m_CellToForgetWeights);
150 this->VerifyConstTensors(
151 "m_CellToOutputWeights", m_InputParams.m_CellToOutputWeights, params.m_CellToOutputWeights);
152 this->VerifyConstTensors(
153 "m_InputGateBias", m_InputParams.m_InputGateBias, params.m_InputGateBias);
154 this->VerifyConstTensors(
155 "m_ForgetGateBias", m_InputParams.m_ForgetGateBias, params.m_ForgetGateBias);
156 this->VerifyConstTensors(
157 "m_CellBias", m_InputParams.m_CellBias, params.m_CellBias);
158 this->VerifyConstTensors(
159 "m_OutputGateBias", m_InputParams.m_OutputGateBias, params.m_OutputGateBias);
160 this->VerifyConstTensors(
161 "m_ProjectionWeights", m_InputParams.m_ProjectionWeights, params.m_ProjectionWeights);
162 this->VerifyConstTensors(
163 "m_ProjectionBias", m_InputParams.m_ProjectionBias, params.m_ProjectionBias);
164 this->VerifyConstTensors(
165 "m_InputLayerNormWeights", m_InputParams.m_InputLayerNormWeights, params.m_InputLayerNormWeights);
166 this->VerifyConstTensors(
167 "m_ForgetLayerNormWeights", m_InputParams.m_ForgetLayerNormWeights, params.m_ForgetLayerNormWeights);
168 this->VerifyConstTensors(
169 "m_CellLayerNormWeights", m_InputParams.m_CellLayerNormWeights, params.m_CellLayerNormWeights);
170 this->VerifyConstTensors(
171 "m_OutputLayerNormWeights", m_InputParams.m_OutputLayerNormWeights, params.m_OutputLayerNormWeights);
172 }
173
174private:
175 armnn::LstmInputParams m_InputParams;
176};
177
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]178TEST_CASE("SerializeDeserializeLstmCifgPeepholeNoProjection")
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]179{
180 armnn::LstmDescriptor descriptor;
181 descriptor.m_ActivationFunc = 4;
182 descriptor.m_ClippingThresProj = 0.0f;
183 descriptor.m_ClippingThresCell = 0.0f;
184 descriptor.m_CifgEnabled = true; // if this is true then we DON'T need to set the OptCifgParams
185 descriptor.m_ProjectionEnabled = false;
186 descriptor.m_PeepholeEnabled = true;
187
188 const uint32_t batchSize = 1;
189 const uint32_t inputSize = 2;
190 const uint32_t numUnits = 4;
191 const uint32_t outputSize = numUnits;
192
193 armnn::TensorInfo inputWeightsInfo1({numUnits, inputSize}, armnn::DataType::Float32);
194 std::vector<float> inputToForgetWeightsData = GenerateRandomData<float>(inputWeightsInfo1.GetNumElements());
195 armnn::ConstTensor inputToForgetWeights(inputWeightsInfo1, inputToForgetWeightsData);
196
197 std::vector<float> inputToCellWeightsData = GenerateRandomData<float>(inputWeightsInfo1.GetNumElements());
198 armnn::ConstTensor inputToCellWeights(inputWeightsInfo1, inputToCellWeightsData);
199
200 std::vector<float> inputToOutputWeightsData = GenerateRandomData<float>(inputWeightsInfo1.GetNumElements());
201 armnn::ConstTensor inputToOutputWeights(inputWeightsInfo1, inputToOutputWeightsData);
202
203 armnn::TensorInfo inputWeightsInfo2({numUnits, outputSize}, armnn::DataType::Float32);
204 std::vector<float> recurrentToForgetWeightsData = GenerateRandomData<float>(inputWeightsInfo2.GetNumElements());
205 armnn::ConstTensor recurrentToForgetWeights(inputWeightsInfo2, recurrentToForgetWeightsData);
206
207 std::vector<float> recurrentToCellWeightsData = GenerateRandomData<float>(inputWeightsInfo2.GetNumElements());
208 armnn::ConstTensor recurrentToCellWeights(inputWeightsInfo2, recurrentToCellWeightsData);
209
210 std::vector<float> recurrentToOutputWeightsData = GenerateRandomData<float>(inputWeightsInfo2.GetNumElements());
211 armnn::ConstTensor recurrentToOutputWeights(inputWeightsInfo2, recurrentToOutputWeightsData);
212
213 armnn::TensorInfo inputWeightsInfo3({numUnits}, armnn::DataType::Float32);
214 std::vector<float> cellToForgetWeightsData = GenerateRandomData<float>(inputWeightsInfo3.GetNumElements());
215 armnn::ConstTensor cellToForgetWeights(inputWeightsInfo3, cellToForgetWeightsData);
216
217 std::vector<float> cellToOutputWeightsData = GenerateRandomData<float>(inputWeightsInfo3.GetNumElements());
218 armnn::ConstTensor cellToOutputWeights(inputWeightsInfo3, cellToOutputWeightsData);
219
220 std::vector<float> forgetGateBiasData(numUnits, 1.0f);
221 armnn::ConstTensor forgetGateBias(inputWeightsInfo3, forgetGateBiasData);
222
223 std::vector<float> cellBiasData(numUnits, 0.0f);
224 armnn::ConstTensor cellBias(inputWeightsInfo3, cellBiasData);
225
226 std::vector<float> outputGateBiasData(numUnits, 0.0f);
227 armnn::ConstTensor outputGateBias(inputWeightsInfo3, outputGateBiasData);
228
229 armnn::LstmInputParams params;
230 params.m_InputToForgetWeights = &inputToForgetWeights;
231 params.m_InputToCellWeights = &inputToCellWeights;
232 params.m_InputToOutputWeights = &inputToOutputWeights;
233 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
234 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
235 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
236 params.m_ForgetGateBias = &forgetGateBias;
237 params.m_CellBias = &cellBias;
238 params.m_OutputGateBias = &outputGateBias;
239 params.m_CellToForgetWeights = &cellToForgetWeights;
240 params.m_CellToOutputWeights = &cellToOutputWeights;
241
242 armnn::INetworkPtr network = armnn::INetwork::Create();
243 armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(0);
244 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(1);
245 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(2);
246 const std::string layerName("lstm");
247 armnn::IConnectableLayer* const lstmLayer = network->AddLstmLayer(descriptor, params, layerName.c_str());
248 armnn::IConnectableLayer* const scratchBuffer = network->AddOutputLayer(0);
249 armnn::IConnectableLayer* const outputStateOut = network->AddOutputLayer(1);
250 armnn::IConnectableLayer* const cellStateOut = network->AddOutputLayer(2);
251 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(3);
252
253 // connect up
254 armnn::TensorInfo inputTensorInfo({ batchSize, inputSize }, armnn::DataType::Float32);
255 armnn::TensorInfo cellStateTensorInfo({ batchSize, numUnits}, armnn::DataType::Float32);
256 armnn::TensorInfo outputStateTensorInfo({ batchSize, outputSize }, armnn::DataType::Float32);
257 armnn::TensorInfo lstmTensorInfoScratchBuff({ batchSize, numUnits * 3 }, armnn::DataType::Float32);
258
259 inputLayer->GetOutputSlot(0).Connect(lstmLayer->GetInputSlot(0));
260 inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
261
262 outputStateIn->GetOutputSlot(0).Connect(lstmLayer->GetInputSlot(1));
263 outputStateIn->GetOutputSlot(0).SetTensorInfo(outputStateTensorInfo);
264
265 cellStateIn->GetOutputSlot(0).Connect(lstmLayer->GetInputSlot(2));
266 cellStateIn->GetOutputSlot(0).SetTensorInfo(cellStateTensorInfo);
267
268 lstmLayer->GetOutputSlot(0).Connect(scratchBuffer->GetInputSlot(0));
269 lstmLayer->GetOutputSlot(0).SetTensorInfo(lstmTensorInfoScratchBuff);
270
271 lstmLayer->GetOutputSlot(1).Connect(outputStateOut->GetInputSlot(0));
272 lstmLayer->GetOutputSlot(1).SetTensorInfo(outputStateTensorInfo);
273
274 lstmLayer->GetOutputSlot(2).Connect(cellStateOut->GetInputSlot(0));
275 lstmLayer->GetOutputSlot(2).SetTensorInfo(cellStateTensorInfo);
276
277 lstmLayer->GetOutputSlot(3).Connect(outputLayer->GetInputSlot(0));
278 lstmLayer->GetOutputSlot(3).SetTensorInfo(outputStateTensorInfo);
279
280 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]281 CHECK(deserializedNetwork);
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]282
283 VerifyLstmLayer<armnn::LstmDescriptor> checker(
284 layerName,
285 {inputTensorInfo, outputStateTensorInfo, cellStateTensorInfo},
286 {lstmTensorInfoScratchBuff, outputStateTensorInfo, cellStateTensorInfo, outputStateTensorInfo},
287 descriptor,
288 params);
289 deserializedNetwork->ExecuteStrategy(checker);
290}
291
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]292TEST_CASE("SerializeDeserializeLstmNoCifgWithPeepholeAndProjection")
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]293{
294 armnn::LstmDescriptor descriptor;
295 descriptor.m_ActivationFunc = 4;
296 descriptor.m_ClippingThresProj = 0.0f;
297 descriptor.m_ClippingThresCell = 0.0f;
298 descriptor.m_CifgEnabled = false; // if this is true then we DON'T need to set the OptCifgParams
299 descriptor.m_ProjectionEnabled = true;
300 descriptor.m_PeepholeEnabled = true;
301
302 const uint32_t batchSize = 2;
303 const uint32_t inputSize = 5;
304 const uint32_t numUnits = 20;
305 const uint32_t outputSize = 16;
306
307 armnn::TensorInfo tensorInfo20x5({numUnits, inputSize}, armnn::DataType::Float32);
308 std::vector<float> inputToInputWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
309 armnn::ConstTensor inputToInputWeights(tensorInfo20x5, inputToInputWeightsData);
310
311 std::vector<float> inputToForgetWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
312 armnn::ConstTensor inputToForgetWeights(tensorInfo20x5, inputToForgetWeightsData);
313
314 std::vector<float> inputToCellWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
315 armnn::ConstTensor inputToCellWeights(tensorInfo20x5, inputToCellWeightsData);
316
317 std::vector<float> inputToOutputWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
318 armnn::ConstTensor inputToOutputWeights(tensorInfo20x5, inputToOutputWeightsData);
319
320 armnn::TensorInfo tensorInfo20({numUnits}, armnn::DataType::Float32);
321 std::vector<float> inputGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
322 armnn::ConstTensor inputGateBias(tensorInfo20, inputGateBiasData);
323
324 std::vector<float> forgetGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
325 armnn::ConstTensor forgetGateBias(tensorInfo20, forgetGateBiasData);
326
327 std::vector<float> cellBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
328 armnn::ConstTensor cellBias(tensorInfo20, cellBiasData);
329
330 std::vector<float> outputGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
331 armnn::ConstTensor outputGateBias(tensorInfo20, outputGateBiasData);
332
333 armnn::TensorInfo tensorInfo20x16({numUnits, outputSize}, armnn::DataType::Float32);
334 std::vector<float> recurrentToInputWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
335 armnn::ConstTensor recurrentToInputWeights(tensorInfo20x16, recurrentToInputWeightsData);
336
337 std::vector<float> recurrentToForgetWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
338 armnn::ConstTensor recurrentToForgetWeights(tensorInfo20x16, recurrentToForgetWeightsData);
339
340 std::vector<float> recurrentToCellWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
341 armnn::ConstTensor recurrentToCellWeights(tensorInfo20x16, recurrentToCellWeightsData);
342
343 std::vector<float> recurrentToOutputWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
344 armnn::ConstTensor recurrentToOutputWeights(tensorInfo20x16, recurrentToOutputWeightsData);
345
346 std::vector<float> cellToInputWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
347 armnn::ConstTensor cellToInputWeights(tensorInfo20, cellToInputWeightsData);
348
349 std::vector<float> cellToForgetWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
350 armnn::ConstTensor cellToForgetWeights(tensorInfo20, cellToForgetWeightsData);
351
352 std::vector<float> cellToOutputWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
353 armnn::ConstTensor cellToOutputWeights(tensorInfo20, cellToOutputWeightsData);
354
355 armnn::TensorInfo tensorInfo16x20({outputSize, numUnits}, armnn::DataType::Float32);
356 std::vector<float> projectionWeightsData = GenerateRandomData<float>(tensorInfo16x20.GetNumElements());
357 armnn::ConstTensor projectionWeights(tensorInfo16x20, projectionWeightsData);
358
359 armnn::TensorInfo tensorInfo16({outputSize}, armnn::DataType::Float32);
360 std::vector<float> projectionBiasData(outputSize, 0.f);
361 armnn::ConstTensor projectionBias(tensorInfo16, projectionBiasData);
362
363 armnn::LstmInputParams params;
364 params.m_InputToForgetWeights = &inputToForgetWeights;
365 params.m_InputToCellWeights = &inputToCellWeights;
366 params.m_InputToOutputWeights = &inputToOutputWeights;
367 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
368 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
369 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
370 params.m_ForgetGateBias = &forgetGateBias;
371 params.m_CellBias = &cellBias;
372 params.m_OutputGateBias = &outputGateBias;
373
374 // additional params because: descriptor.m_CifgEnabled = false
375 params.m_InputToInputWeights = &inputToInputWeights;
376 params.m_RecurrentToInputWeights = &recurrentToInputWeights;
377 params.m_CellToInputWeights = &cellToInputWeights;
378 params.m_InputGateBias = &inputGateBias;
379
380 // additional params because: descriptor.m_ProjectionEnabled = true
381 params.m_ProjectionWeights = &projectionWeights;
382 params.m_ProjectionBias = &projectionBias;
383
384 // additional params because: descriptor.m_PeepholeEnabled = true
385 params.m_CellToForgetWeights = &cellToForgetWeights;
386 params.m_CellToOutputWeights = &cellToOutputWeights;
387
388 armnn::INetworkPtr network = armnn::INetwork::Create();
389 armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(0);
390 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(1);
391 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(2);
392 const std::string layerName("lstm");
393 armnn::IConnectableLayer* const lstmLayer = network->AddLstmLayer(descriptor, params, layerName.c_str());
394 armnn::IConnectableLayer* const scratchBuffer = network->AddOutputLayer(0);
395 armnn::IConnectableLayer* const outputStateOut = network->AddOutputLayer(1);
396 armnn::IConnectableLayer* const cellStateOut = network->AddOutputLayer(2);
397 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(3);
398
399 // connect up
400 armnn::TensorInfo inputTensorInfo({ batchSize, inputSize }, armnn::DataType::Float32);
401 armnn::TensorInfo cellStateTensorInfo({ batchSize, numUnits}, armnn::DataType::Float32);
402 armnn::TensorInfo outputStateTensorInfo({ batchSize, outputSize }, armnn::DataType::Float32);
403 armnn::TensorInfo lstmTensorInfoScratchBuff({ batchSize, numUnits * 4 }, armnn::DataType::Float32);
404
405 inputLayer->GetOutputSlot(0).Connect(lstmLayer->GetInputSlot(0));
406 inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
407
408 outputStateIn->GetOutputSlot(0).Connect(lstmLayer->GetInputSlot(1));
409 outputStateIn->GetOutputSlot(0).SetTensorInfo(outputStateTensorInfo);
410
411 cellStateIn->GetOutputSlot(0).Connect(lstmLayer->GetInputSlot(2));
412 cellStateIn->GetOutputSlot(0).SetTensorInfo(cellStateTensorInfo);
413
414 lstmLayer->GetOutputSlot(0).Connect(scratchBuffer->GetInputSlot(0));
415 lstmLayer->GetOutputSlot(0).SetTensorInfo(lstmTensorInfoScratchBuff);
416
417 lstmLayer->GetOutputSlot(1).Connect(outputStateOut->GetInputSlot(0));
418 lstmLayer->GetOutputSlot(1).SetTensorInfo(outputStateTensorInfo);
419
420 lstmLayer->GetOutputSlot(2).Connect(cellStateOut->GetInputSlot(0));
421 lstmLayer->GetOutputSlot(2).SetTensorInfo(cellStateTensorInfo);
422
423 lstmLayer->GetOutputSlot(3).Connect(outputLayer->GetInputSlot(0));
424 lstmLayer->GetOutputSlot(3).SetTensorInfo(outputStateTensorInfo);
425
426 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]427 CHECK(deserializedNetwork);
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]428
429 VerifyLstmLayer<armnn::LstmDescriptor> checker(
430 layerName,
431 {inputTensorInfo, outputStateTensorInfo, cellStateTensorInfo},
432 {lstmTensorInfoScratchBuff, outputStateTensorInfo, cellStateTensorInfo, outputStateTensorInfo},
433 descriptor,
434 params);
435 deserializedNetwork->ExecuteStrategy(checker);
436}
437
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]438TEST_CASE("SerializeDeserializeLstmNoCifgWithPeepholeWithProjectionWithLayerNorm")
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]439{
440 armnn::LstmDescriptor descriptor;
441 descriptor.m_ActivationFunc = 4;
442 descriptor.m_ClippingThresProj = 0.0f;
443 descriptor.m_ClippingThresCell = 0.0f;
444 descriptor.m_CifgEnabled = false; // if this is true then we DON'T need to set the OptCifgParams
445 descriptor.m_ProjectionEnabled = true;
446 descriptor.m_PeepholeEnabled = true;
447 descriptor.m_LayerNormEnabled = true;
448
449 const uint32_t batchSize = 2;
450 const uint32_t inputSize = 5;
451 const uint32_t numUnits = 20;
452 const uint32_t outputSize = 16;
453
454 armnn::TensorInfo tensorInfo20x5({numUnits, inputSize}, armnn::DataType::Float32);
455 std::vector<float> inputToInputWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
456 armnn::ConstTensor inputToInputWeights(tensorInfo20x5, inputToInputWeightsData);
457
458 std::vector<float> inputToForgetWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
459 armnn::ConstTensor inputToForgetWeights(tensorInfo20x5, inputToForgetWeightsData);
460
461 std::vector<float> inputToCellWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
462 armnn::ConstTensor inputToCellWeights(tensorInfo20x5, inputToCellWeightsData);
463
464 std::vector<float> inputToOutputWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
465 armnn::ConstTensor inputToOutputWeights(tensorInfo20x5, inputToOutputWeightsData);
466
467 armnn::TensorInfo tensorInfo20({numUnits}, armnn::DataType::Float32);
468 std::vector<float> inputGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
469 armnn::ConstTensor inputGateBias(tensorInfo20, inputGateBiasData);
470
471 std::vector<float> forgetGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
472 armnn::ConstTensor forgetGateBias(tensorInfo20, forgetGateBiasData);
473
474 std::vector<float> cellBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
475 armnn::ConstTensor cellBias(tensorInfo20, cellBiasData);
476
477 std::vector<float> outputGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
478 armnn::ConstTensor outputGateBias(tensorInfo20, outputGateBiasData);
479
480 armnn::TensorInfo tensorInfo20x16({numUnits, outputSize}, armnn::DataType::Float32);
481 std::vector<float> recurrentToInputWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
482 armnn::ConstTensor recurrentToInputWeights(tensorInfo20x16, recurrentToInputWeightsData);
483
484 std::vector<float> recurrentToForgetWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
485 armnn::ConstTensor recurrentToForgetWeights(tensorInfo20x16, recurrentToForgetWeightsData);
486
487 std::vector<float> recurrentToCellWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
488 armnn::ConstTensor recurrentToCellWeights(tensorInfo20x16, recurrentToCellWeightsData);
489
490 std::vector<float> recurrentToOutputWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
491 armnn::ConstTensor recurrentToOutputWeights(tensorInfo20x16, recurrentToOutputWeightsData);
492
493 std::vector<float> cellToInputWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
494 armnn::ConstTensor cellToInputWeights(tensorInfo20, cellToInputWeightsData);
495
496 std::vector<float> cellToForgetWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
497 armnn::ConstTensor cellToForgetWeights(tensorInfo20, cellToForgetWeightsData);
498
499 std::vector<float> cellToOutputWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
500 armnn::ConstTensor cellToOutputWeights(tensorInfo20, cellToOutputWeightsData);
501
502 armnn::TensorInfo tensorInfo16x20({outputSize, numUnits}, armnn::DataType::Float32);
503 std::vector<float> projectionWeightsData = GenerateRandomData<float>(tensorInfo16x20.GetNumElements());
504 armnn::ConstTensor projectionWeights(tensorInfo16x20, projectionWeightsData);
505
506 armnn::TensorInfo tensorInfo16({outputSize}, armnn::DataType::Float32);
507 std::vector<float> projectionBiasData(outputSize, 0.f);
508 armnn::ConstTensor projectionBias(tensorInfo16, projectionBiasData);
509
510 std::vector<float> inputLayerNormWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
511 armnn::ConstTensor inputLayerNormWeights(tensorInfo20, forgetGateBiasData);
512
513 std::vector<float> forgetLayerNormWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
514 armnn::ConstTensor forgetLayerNormWeights(tensorInfo20, forgetGateBiasData);
515
516 std::vector<float> cellLayerNormWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
517 armnn::ConstTensor cellLayerNormWeights(tensorInfo20, forgetGateBiasData);
518
519 std::vector<float> outLayerNormWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
520 armnn::ConstTensor outLayerNormWeights(tensorInfo20, forgetGateBiasData);
521
522 armnn::LstmInputParams params;
523 params.m_InputToForgetWeights = &inputToForgetWeights;
524 params.m_InputToCellWeights = &inputToCellWeights;
525 params.m_InputToOutputWeights = &inputToOutputWeights;
526 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
527 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
528 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
529 params.m_ForgetGateBias = &forgetGateBias;
530 params.m_CellBias = &cellBias;
531 params.m_OutputGateBias = &outputGateBias;
532
533 // additional params because: descriptor.m_CifgEnabled = false
534 params.m_InputToInputWeights = &inputToInputWeights;
535 params.m_RecurrentToInputWeights = &recurrentToInputWeights;
536 params.m_CellToInputWeights = &cellToInputWeights;
537 params.m_InputGateBias = &inputGateBias;
538
539 // additional params because: descriptor.m_ProjectionEnabled = true
540 params.m_ProjectionWeights = &projectionWeights;
541 params.m_ProjectionBias = &projectionBias;
542
543 // additional params because: descriptor.m_PeepholeEnabled = true
544 params.m_CellToForgetWeights = &cellToForgetWeights;
545 params.m_CellToOutputWeights = &cellToOutputWeights;
546
547 // additional params because: despriptor.m_LayerNormEnabled = true
548 params.m_InputLayerNormWeights = &inputLayerNormWeights;
549 params.m_ForgetLayerNormWeights = &forgetLayerNormWeights;
550 params.m_CellLayerNormWeights = &cellLayerNormWeights;
551 params.m_OutputLayerNormWeights = &outLayerNormWeights;
552
553 armnn::INetworkPtr network = armnn::INetwork::Create();
554 armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(0);
555 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(1);
556 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(2);
557 const std::string layerName("lstm");
558 armnn::IConnectableLayer* const lstmLayer = network->AddLstmLayer(descriptor, params, layerName.c_str());
559 armnn::IConnectableLayer* const scratchBuffer = network->AddOutputLayer(0);
560 armnn::IConnectableLayer* const outputStateOut = network->AddOutputLayer(1);
561 armnn::IConnectableLayer* const cellStateOut = network->AddOutputLayer(2);
562 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(3);
563
564 // connect up
565 armnn::TensorInfo inputTensorInfo({ batchSize, inputSize }, armnn::DataType::Float32);
566 armnn::TensorInfo cellStateTensorInfo({ batchSize, numUnits}, armnn::DataType::Float32);
567 armnn::TensorInfo outputStateTensorInfo({ batchSize, outputSize }, armnn::DataType::Float32);
568 armnn::TensorInfo lstmTensorInfoScratchBuff({ batchSize, numUnits * 4 }, armnn::DataType::Float32);
569
570 inputLayer->GetOutputSlot(0).Connect(lstmLayer->GetInputSlot(0));
571 inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
572
573 outputStateIn->GetOutputSlot(0).Connect(lstmLayer->GetInputSlot(1));
574 outputStateIn->GetOutputSlot(0).SetTensorInfo(outputStateTensorInfo);
575
576 cellStateIn->GetOutputSlot(0).Connect(lstmLayer->GetInputSlot(2));
577 cellStateIn->GetOutputSlot(0).SetTensorInfo(cellStateTensorInfo);
578
579 lstmLayer->GetOutputSlot(0).Connect(scratchBuffer->GetInputSlot(0));
580 lstmLayer->GetOutputSlot(0).SetTensorInfo(lstmTensorInfoScratchBuff);
581
582 lstmLayer->GetOutputSlot(1).Connect(outputStateOut->GetInputSlot(0));
583 lstmLayer->GetOutputSlot(1).SetTensorInfo(outputStateTensorInfo);
584
585 lstmLayer->GetOutputSlot(2).Connect(cellStateOut->GetInputSlot(0));
586 lstmLayer->GetOutputSlot(2).SetTensorInfo(cellStateTensorInfo);
587
588 lstmLayer->GetOutputSlot(3).Connect(outputLayer->GetInputSlot(0));
589 lstmLayer->GetOutputSlot(3).SetTensorInfo(outputStateTensorInfo);
590
591 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]592 CHECK(deserializedNetwork);
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]593
594 VerifyLstmLayer<armnn::LstmDescriptor> checker(
595 layerName,
596 {inputTensorInfo, outputStateTensorInfo, cellStateTensorInfo},
597 {lstmTensorInfoScratchBuff, outputStateTensorInfo, cellStateTensorInfo, outputStateTensorInfo},
598 descriptor,
599 params);
600 deserializedNetwork->ExecuteStrategy(checker);
601}
602
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]603TEST_CASE("EnsureLstmLayersBackwardCompatibility")
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]604{
605 // The hex data below is a flat buffer containing a lstm layer with no Cifg, with peephole and projection
606 // enabled. That data was obtained before additional layer normalization parameters where added to the
607 // lstm serializer. That way it can be tested if a lstm model with the old parameter configuration can
608 // still be loaded
609 const std::vector<uint8_t> lstmNoCifgWithPeepholeAndProjectionModel =
610 {
611 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x10, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0C, 0x00, 0x0A, 0x00,
612 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
613 0xDC, 0x29, 0x00, 0x00, 0x38, 0x29, 0x00, 0x00, 0xB4, 0x28, 0x00, 0x00, 0x94, 0x01, 0x00, 0x00, 0x3C, 0x01,
614 0x00, 0x00, 0xE0, 0x00, 0x00, 0x00, 0x84, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00,
615 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00,
616 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x70, 0xD6, 0xFF, 0xFF,
617 0x00, 0x00, 0x00, 0x0B, 0x04, 0x00, 0x00, 0x00, 0x06, 0xD7, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x88, 0xD7,
618 0xFF, 0xFF, 0x08, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xF6, 0xD6, 0xFF, 0xFF, 0x07, 0x00, 0x00, 0x00,
619 0x10, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00,
620 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
621 0xE8, 0xD7, 0xFF, 0xFF, 0x03, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xC8, 0xD6, 0xFF, 0xFF, 0x00, 0x00,
622 0x00, 0x0B, 0x04, 0x00, 0x00, 0x00, 0x5E, 0xD7, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0xE0, 0xD7, 0xFF, 0xFF,
623 0x08, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x4E, 0xD7, 0xFF, 0xFF, 0x06, 0x00, 0x00, 0x00, 0x10, 0x00,
624 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
625 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0xD8,
626 0xFF, 0xFF, 0x03, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x20, 0xD7, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x0B,
627 0x04, 0x00, 0x00, 0x00, 0xB6, 0xD7, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x38, 0xD8, 0xFF, 0xFF, 0x08, 0x00,
628 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xA6, 0xD7, 0xFF, 0xFF, 0x05, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,
629 0x03, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
630 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0xD8, 0xFF, 0xFF,
631 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x78, 0xD7, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x0B, 0x04, 0x00,
632 0x00, 0x00, 0x0E, 0xD8, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x16, 0xD8, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00,
633 0xFA, 0xD7, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x10, 0x00,
634 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
635 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xD8, 0xFF, 0xFF, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
636 0x00, 0x00, 0x6C, 0xD8, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x23, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0A, 0x00,
637 0x12, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0C, 0x00, 0x0A, 0x00, 0x00, 0x00, 0xE0, 0x25, 0x00, 0x00, 0xD0, 0x25,
638 0x00, 0x00, 0x2C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x00, 0x48, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0C, 0x00,
639 0x10, 0x00, 0x14, 0x00, 0x18, 0x00, 0x1C, 0x00, 0x20, 0x00, 0x24, 0x00, 0x28, 0x00, 0x2C, 0x00, 0x30, 0x00,
640 0x34, 0x00, 0x38, 0x00, 0x3C, 0x00, 0x40, 0x00, 0x44, 0x00, 0x26, 0x00, 0x00, 0x00, 0xC4, 0x23, 0x00, 0x00,
641 0xF8, 0x21, 0x00, 0x00, 0x2C, 0x20, 0x00, 0x00, 0xF0, 0x1A, 0x00, 0x00, 0xB4, 0x15, 0x00, 0x00, 0x78, 0x10,
642 0x00, 0x00, 0xF0, 0x0F, 0x00, 0x00, 0x68, 0x0F, 0x00, 0x00, 0xE0, 0x0E, 0x00, 0x00, 0x14, 0x0D, 0x00, 0x00,
643 0xD8, 0x07, 0x00, 0x00, 0x50, 0x07, 0x00, 0x00, 0xC8, 0x06, 0x00, 0x00, 0x8C, 0x01, 0x00, 0x00, 0x14, 0x01,
644 0x00, 0x00, 0x8C, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xEE, 0xD7, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03,
645 0x64, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xFE, 0xD8, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x14, 0x00,
646 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
647 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
648 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
649 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
650 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5A, 0xD8, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01,
651 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x72, 0xD8,
652 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0x64, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x82, 0xD9, 0xFF, 0xFF,
653 0x04, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
654 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
655 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
656 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
657 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xD8,
658 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
659 0x14, 0x00, 0x00, 0x00, 0xF6, 0xD8, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0x54, 0x00, 0x00, 0x00, 0x04, 0x00,
660 0x00, 0x00, 0x06, 0xDA, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
661 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
662 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
663 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
664 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0xD9, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00,
665 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x6A, 0xD9, 0xFF, 0xFF, 0x00, 0x00,
666 0x00, 0x03, 0x14, 0x05, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7A, 0xDA, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00,
667 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
668 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
669 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
670 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
671 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
672 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
673 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
674 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
675 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
676 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
677 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
678 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
679 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
680 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
681 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
682 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
683 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
684 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
685 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
686 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
687 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
688 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
689 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
690 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
691 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
692 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
693 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
694 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
695 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
696 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
697 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
698 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
699 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
700 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
701 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
702 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
703 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
704 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
705 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
706 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
707 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
708 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
709 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
710 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
711 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
712 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
713 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
714 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
715 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
716 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
717 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
718 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
719 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
720 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
721 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
722 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
723 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
724 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
725 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
726 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
727 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
728 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
729 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
730 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
731 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
732 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
733 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
734 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
735 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
736 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
737 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
738 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0xDE, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00,
739 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0xA2, 0xDE,
740 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0x64, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xB2, 0xDF, 0xFF, 0xFF,
741 0x04, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
742 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
743 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
744 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
745 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0E, 0xDF,
746 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
747 0x14, 0x00, 0x00, 0x00, 0x26, 0xDF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0x64, 0x00, 0x00, 0x00, 0x04, 0x00,
748 0x00, 0x00, 0x36, 0xE0, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
749 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
750 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
751 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
752 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
753 0x00, 0x00, 0x00, 0x00, 0x92, 0xDF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
754 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0xAA, 0xDF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03,
755 0x14, 0x05, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0xE0, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x40, 0x01,
756 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
757 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
758 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
759 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
760 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
761 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
762 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
763 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
764 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
765 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
766 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
767 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
768 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
769 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
770 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
771 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
772 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
773 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
774 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
775 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
776 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
777 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
778 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
779 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
780 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
781 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
782 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
783 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
784 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
785 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
786 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
787 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
788 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
789 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
790 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
791 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
792 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
793 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
794 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
795 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
796 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
797 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
798 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
799 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
800 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
801 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
802 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
803 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
804 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
805 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
806 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
807 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
808 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
809 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
810 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
811 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
812 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
813 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
814 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
815 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
816 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
817 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
818 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
819 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
820 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
821 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
822 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
823 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
824 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
825 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
826 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
827 0x00, 0x00, 0x00, 0x00, 0xC6, 0xE4, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
828 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xE2, 0xE4, 0xFF, 0xFF,
829 0x00, 0x00, 0x00, 0x03, 0xA4, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xF2, 0xE5, 0xFF, 0xFF, 0x04, 0x00,
830 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
831 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
832 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
833 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
834 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
835 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
836 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
837 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
838 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
839 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
840 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
841 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
842 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
843 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
844 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
845 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
846 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
847 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
848 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
849 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
850 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
851 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
852 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0xE6, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01,
853 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x05, 0x00,
854 0x00, 0x00, 0xAA, 0xE6, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0x64, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
855 0xBA, 0xE7, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
856 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
857 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
858 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
859 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
860 0x00, 0x00, 0x16, 0xE7, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
861 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x2E, 0xE7, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0x64, 0x00,
862 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x3E, 0xE8, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00,
863 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
864 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
865 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
866 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
867 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0xE7, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00,
868 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0xB2, 0xE7, 0xFF, 0xFF,
869 0x00, 0x00, 0x00, 0x03, 0x64, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xC2, 0xE8, 0xFF, 0xFF, 0x04, 0x00,
870 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
871 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
872 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
873 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
874 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1E, 0xE8, 0xFF, 0xFF,
875 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00,
876 0x00, 0x00, 0x36, 0xE8, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0x14, 0x05, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
877 0x46, 0xE9, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
878 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
879 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
880 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
881 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
882 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
883 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
884 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
885 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
886 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
887 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
888 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
889 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
890 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
891 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
892 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
893 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
894 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
895 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
896 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
897 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
898 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
899 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
900 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
901 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
902 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
903 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
904 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
905 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
906 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
907 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
908 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
909 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
910 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
911 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
912 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
913 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
914 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
915 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
916 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
917 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
918 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
919 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
920 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
921 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
922 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
923 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
924 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
925 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
926 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
927 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
928 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
929 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
930 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
931 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
932 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
933 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
934 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
935 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
936 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
937 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
938 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
939 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
940 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
941 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
942 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
943 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
944 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
945 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
946 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
947 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
948 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0xED, 0xFF, 0xFF,
949 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00,
950 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x6E, 0xED, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0x14, 0x05, 0x00, 0x00,
951 0x04, 0x00, 0x00, 0x00, 0x7E, 0xEE, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00,
952 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
953 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
954 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
955 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
956 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
957 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
958 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
959 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
960 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
961 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
962 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
963 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
964 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
965 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
966 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
967 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
968 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
969 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
970 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
971 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
972 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
973 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
974 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
975 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
976 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
977 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
978 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
979 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
980 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
981 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
982 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
983 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
984 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
985 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
986 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
987 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
988 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
989 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
990 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
991 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
992 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
993 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
994 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
995 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
996 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
997 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
998 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
999 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1000 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1001 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1002 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1003 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1004 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1005 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1006 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1007 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1008 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1009 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1010 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1011 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1012 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1013 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1014 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1015 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1016 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1017 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1018 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1019 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1020 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1021 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1022 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1023 0x8A, 0xF2, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00,
1024 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xA6, 0xF2, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03,
1025 0x14, 0x05, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xB6, 0xF3, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x40, 0x01,
1026 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1027 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1028 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1029 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1030 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1031 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1032 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1033 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1034 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1035 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1036 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1037 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1038 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1039 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1040 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1041 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1042 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1043 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1044 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1045 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1046 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1047 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1048 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1049 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1050 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1051 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1052 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1053 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1054 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1055 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1056 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1057 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1058 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1059 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1060 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1061 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1062 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1063 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1064 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1065 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1066 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1067 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1068 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1069 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1070 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1071 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1072 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1073 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1074 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1075 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1076 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1077 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1078 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1079 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1080 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1081 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1082 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1083 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1084 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1085 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1086 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1087 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1088 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1089 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1090 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1091 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1092 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1093 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1094 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1095 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1096 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1097 0x00, 0x00, 0x00, 0x00, 0xC2, 0xF7, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
1098 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xDE, 0xF7, 0xFF, 0xFF,
1099 0x00, 0x00, 0x00, 0x03, 0xA4, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xEE, 0xF8, 0xFF, 0xFF, 0x04, 0x00,
1100 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1101 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1102 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1103 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1104 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1105 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1106 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1107 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1108 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1109 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1110 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1111 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1112 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1113 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1114 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1115 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1116 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1117 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1118 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1119 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1120 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1121 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1122 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xF9, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01,
1123 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x05, 0x00,
1124 0x00, 0x00, 0xA6, 0xF9, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0xA4, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
1125 0xB6, 0xFA, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1126 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1127 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1128 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1129 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1130 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1131 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1132 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1133 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1134 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1135 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1136 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1137 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1138 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1139 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1140 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1141 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1142 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1143 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1144 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1145 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1146 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1147 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, 0xFB,
1148 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
1149 0x14, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x6E, 0xFB, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x03, 0xA4, 0x01,
1150 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7E, 0xFC, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00,
1151 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1152 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1153 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1154 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1155 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1156 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1157 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1158 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1159 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1160 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1161 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1162 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1163 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1164 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1165 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1166 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1167 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1168 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1169 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1170 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1171 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1172 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1173 0x00, 0x00, 0x00, 0x00, 0x1A, 0xFD, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
1174 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x10, 0x00, 0x0C, 0x00,
1175 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x06, 0x00, 0x07, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,
1176 0x01, 0x01, 0x04, 0x00, 0x00, 0x00, 0x2E, 0xFE, 0xFF, 0xFF, 0x03, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,
1177 0x22, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6C, 0x73,
1178 0x74, 0x6D, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xEC, 0x00, 0x00, 0x00, 0xD0, 0x00, 0x00, 0x00,
1179 0xB4, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x5C, 0x00, 0x00, 0x00, 0x30, 0x00,
1180 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x14, 0xFF, 0xFF, 0xFF, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
1181 0xA6, 0xFD, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00,
1182 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x3C, 0xFF, 0xFF, 0xFF, 0x02, 0x00, 0x00, 0x00,
1183 0x04, 0x00, 0x00, 0x00, 0xCE, 0xFD, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
1184 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x64, 0xFF, 0xFF, 0xFF,
1185 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xF6, 0xFD, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00,
1186 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,
1187 0xB4, 0xFE, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0x1A, 0xFE, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00,
1188 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00,
1189 0xF0, 0xFF, 0xFF, 0xFF, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00,
1190 0x10, 0x00, 0x04, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
1191 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
1192 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xFE, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x09, 0x04, 0x00, 0x00, 0x00,
1193 0x7E, 0xFF, 0xFF, 0xFF, 0x0C, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0C, 0x00, 0x04, 0x00, 0x08, 0x00, 0x08, 0x00,
1194 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x76, 0xFF, 0xFF, 0xFF, 0x02, 0x00, 0x00, 0x00,
1195 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,
1196 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
1197 0x68, 0xFF, 0xFF, 0xFF, 0x04, 0x00, 0x00, 0x00, 0xCE, 0xFE, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00,
1198 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,
1199 0x08, 0x00, 0x0E, 0x00, 0x07, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x0C, 0x00,
1200 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x08, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00,
1201 0x08, 0x00, 0x0E, 0x00, 0x04, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x01, 0x00,
1202 0x00, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x18, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0C, 0x00, 0x10, 0x00, 0x14, 0x00,
1203 0x0E, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00,
1204 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1205 0x01, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00,
1206 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6E, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00,
1207 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x08, 0x00,
1208 0x0C, 0x00, 0x07, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x04, 0x00, 0x00, 0x00,
1209 0xF6, 0xFF, 0xFF, 0xFF, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x0A, 0x00, 0x04, 0x00, 0x06, 0x00,
1210 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00,
1211 0x0C, 0x00, 0x10, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00,
1212 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1213 0x01, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00,
1214 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x10, 0x00, 0x08, 0x00, 0x07, 0x00, 0x0C, 0x00,
1215 0x0A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00,
1216 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00
1217 };
1218
1219 armnn::INetworkPtr deserializedNetwork =
1220 DeserializeNetwork(std::string(lstmNoCifgWithPeepholeAndProjectionModel.begin(),
1221 lstmNoCifgWithPeepholeAndProjectionModel.end()));
1222
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]1223 CHECK(deserializedNetwork);
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]1224
1225 // generating the same model parameters which where used to serialize the model (Layer norm is not specified)
1226 armnn::LstmDescriptor descriptor;
1227 descriptor.m_ActivationFunc = 4;
1228 descriptor.m_ClippingThresProj = 0.0f;
1229 descriptor.m_ClippingThresCell = 0.0f;
1230 descriptor.m_CifgEnabled = false;
1231 descriptor.m_ProjectionEnabled = true;
1232 descriptor.m_PeepholeEnabled = true;
1233
1234 const uint32_t batchSize = 2u;
1235 const uint32_t inputSize = 5u;
1236 const uint32_t numUnits = 20u;
1237 const uint32_t outputSize = 16u;
1238
1239 armnn::TensorInfo tensorInfo20x5({numUnits, inputSize}, armnn::DataType::Float32);
1240 std::vector<float> inputToInputWeightsData(tensorInfo20x5.GetNumElements(), 0.0f);
1241 armnn::ConstTensor inputToInputWeights(tensorInfo20x5, inputToInputWeightsData);
1242
1243 std::vector<float> inputToForgetWeightsData(tensorInfo20x5.GetNumElements(), 0.0f);
1244 armnn::ConstTensor inputToForgetWeights(tensorInfo20x5, inputToForgetWeightsData);
1245
1246 std::vector<float> inputToCellWeightsData(tensorInfo20x5.GetNumElements(), 0.0f);
1247 armnn::ConstTensor inputToCellWeights(tensorInfo20x5, inputToCellWeightsData);
1248
1249 std::vector<float> inputToOutputWeightsData(tensorInfo20x5.GetNumElements(), 0.0f);
1250 armnn::ConstTensor inputToOutputWeights(tensorInfo20x5, inputToOutputWeightsData);
1251
1252 armnn::TensorInfo tensorInfo20({numUnits}, armnn::DataType::Float32);
1253 std::vector<float> inputGateBiasData(tensorInfo20.GetNumElements(), 0.0f);
1254 armnn::ConstTensor inputGateBias(tensorInfo20, inputGateBiasData);
1255
1256 std::vector<float> forgetGateBiasData(tensorInfo20.GetNumElements(), 0.0f);
1257 armnn::ConstTensor forgetGateBias(tensorInfo20, forgetGateBiasData);
1258
1259 std::vector<float> cellBiasData(tensorInfo20.GetNumElements(), 0.0f);
1260 armnn::ConstTensor cellBias(tensorInfo20, cellBiasData);
1261
1262 std::vector<float> outputGateBiasData(tensorInfo20.GetNumElements(), 0.0f);
1263 armnn::ConstTensor outputGateBias(tensorInfo20, outputGateBiasData);
1264
1265 armnn::TensorInfo tensorInfo20x16({numUnits, outputSize}, armnn::DataType::Float32);
1266 std::vector<float> recurrentToInputWeightsData(tensorInfo20x16.GetNumElements(), 0.0f);
1267 armnn::ConstTensor recurrentToInputWeights(tensorInfo20x16, recurrentToInputWeightsData);
1268
1269 std::vector<float> recurrentToForgetWeightsData(tensorInfo20x16.GetNumElements(), 0.0f);
1270 armnn::ConstTensor recurrentToForgetWeights(tensorInfo20x16, recurrentToForgetWeightsData);
1271
1272 std::vector<float> recurrentToCellWeightsData(tensorInfo20x16.GetNumElements(), 0.0f);
1273 armnn::ConstTensor recurrentToCellWeights(tensorInfo20x16, recurrentToCellWeightsData);
1274
1275 std::vector<float> recurrentToOutputWeightsData(tensorInfo20x16.GetNumElements(), 0.0f);
1276 armnn::ConstTensor recurrentToOutputWeights(tensorInfo20x16, recurrentToOutputWeightsData);
1277
1278 std::vector<float> cellToInputWeightsData(tensorInfo20.GetNumElements(), 0.0f);
1279 armnn::ConstTensor cellToInputWeights(tensorInfo20, cellToInputWeightsData);
1280
1281 std::vector<float> cellToForgetWeightsData(tensorInfo20.GetNumElements(), 0.0f);
1282 armnn::ConstTensor cellToForgetWeights(tensorInfo20, cellToForgetWeightsData);
1283
1284 std::vector<float> cellToOutputWeightsData(tensorInfo20.GetNumElements(), 0.0f);
1285 armnn::ConstTensor cellToOutputWeights(tensorInfo20, cellToOutputWeightsData);
1286
1287 armnn::TensorInfo tensorInfo16x20({outputSize, numUnits}, armnn::DataType::Float32);
1288 std::vector<float> projectionWeightsData(tensorInfo16x20.GetNumElements(), 0.0f);
1289 armnn::ConstTensor projectionWeights(tensorInfo16x20, projectionWeightsData);
1290
1291 armnn::TensorInfo tensorInfo16({outputSize}, armnn::DataType::Float32);
1292 std::vector<float> projectionBiasData(outputSize, 0.0f);
1293 armnn::ConstTensor projectionBias(tensorInfo16, projectionBiasData);
1294
1295 armnn::LstmInputParams params;
1296 params.m_InputToForgetWeights = &inputToForgetWeights;
1297 params.m_InputToCellWeights = &inputToCellWeights;
1298 params.m_InputToOutputWeights = &inputToOutputWeights;
1299 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
1300 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
1301 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
1302 params.m_ForgetGateBias = &forgetGateBias;
1303 params.m_CellBias = &cellBias;
1304 params.m_OutputGateBias = &outputGateBias;
1305
1306 // additional params because: descriptor.m_CifgEnabled = false
1307 params.m_InputToInputWeights = &inputToInputWeights;
1308 params.m_RecurrentToInputWeights = &recurrentToInputWeights;
1309 params.m_CellToInputWeights = &cellToInputWeights;
1310 params.m_InputGateBias = &inputGateBias;
1311
1312 // additional params because: descriptor.m_ProjectionEnabled = true
1313 params.m_ProjectionWeights = &projectionWeights;
1314 params.m_ProjectionBias = &projectionBias;
1315
1316 // additional params because: descriptor.m_PeepholeEnabled = true
1317 params.m_CellToForgetWeights = &cellToForgetWeights;
1318 params.m_CellToOutputWeights = &cellToOutputWeights;
1319
1320 const std::string layerName("lstm");
1321 armnn::TensorInfo inputTensorInfo({ batchSize, inputSize }, armnn::DataType::Float32);
1322 armnn::TensorInfo cellStateTensorInfo({ batchSize, numUnits}, armnn::DataType::Float32);
1323 armnn::TensorInfo outputStateTensorInfo({ batchSize, outputSize }, armnn::DataType::Float32);
1324 armnn::TensorInfo lstmTensorInfoScratchBuff({ batchSize, numUnits * 4 }, armnn::DataType::Float32);
1325
1326 VerifyLstmLayer<armnn::LstmDescriptor> checker(
1327 layerName,
1328 {inputTensorInfo, outputStateTensorInfo, cellStateTensorInfo},
1329 {lstmTensorInfoScratchBuff, outputStateTensorInfo, cellStateTensorInfo, outputStateTensorInfo},
1330 descriptor,
1331 params);
1332 deserializedNetwork->ExecuteStrategy(checker);
1333}
1334
1335armnn::QuantizedLstmInputParams ConstantsVector2QuantizedLstmInputParams(
1336 const std::vector<armnn::ConstTensor>& constants)
1337{
1338 armnn::QuantizedLstmInputParams params;
1339
1340 // index for constants vector
1341 size_t i = 0;
1342
1343 // Get input parameters
1344 params.m_InputToInputWeights = &constants[i++];
1345 params.m_InputToForgetWeights = &constants[i++];
1346 params.m_InputToCellWeights = &constants[i++];
1347 params.m_InputToOutputWeights = &constants[i++];
1348
1349 params.m_RecurrentToInputWeights = &constants[i++];
1350 params.m_RecurrentToForgetWeights = &constants[i++];
1351 params.m_RecurrentToCellWeights = &constants[i++];
1352 params.m_RecurrentToOutputWeights = &constants[i++];
1353
1354 params.m_InputGateBias = &constants[i++];
1355 params.m_ForgetGateBias = &constants[i++];
1356 params.m_CellBias = &constants[i++];
1357 params.m_OutputGateBias = &constants[i++];
1358
1359 return params;
1360}
1361
1362class VerifyQuantizedLstmLayer : public LayerVerifierBase
1363{
1364
1365public:
1366 VerifyQuantizedLstmLayer(const std::string& layerName,
1367 const std::vector<armnn::TensorInfo>& inputInfos,
1368 const std::vector<armnn::TensorInfo>& outputInfos,
1369 const armnn::QuantizedLstmInputParams& inputParams)
1370 : LayerVerifierBase(layerName, inputInfos, outputInfos), m_InputParams(inputParams) {}
1371
1372 void ExecuteStrategy(const armnn::IConnectableLayer* layer,
1373 const armnn::BaseDescriptor& descriptor,
1374 const std::vector<armnn::ConstTensor>& constants,
1375 const char* name,
1376 const armnn::LayerBindingId id = 0) override
1377 {
1378 armnn::IgnoreUnused(descriptor, constants, id);
1379 switch (layer->GetType())
1380 {
1381 case armnn::LayerType::Input: break;
1382 case armnn::LayerType::Output: break;
1383 case armnn::LayerType::QuantizedLstm:
1384 {
1385 VerifyNameAndConnections(layer, name);
1386 armnn::QuantizedLstmInputParams params = ConstantsVector2QuantizedLstmInputParams(constants);
1387 VerifyInputParameters(params);
1388 break;
1389 }
1390 default:
1391 {
1392 throw armnn::Exception(fmt::format("Unexpected layer type in QuantizedLstm test model:",
1393 layer->GetName()));
1394 }
1395 }
1396 }
1397
1398protected:
1399 void VerifyInputParameters(const armnn::QuantizedLstmInputParams& params)
1400 {
1401 VerifyConstTensors("m_InputToInputWeights",
1402 m_InputParams.m_InputToInputWeights, params.m_InputToInputWeights);
1403 VerifyConstTensors("m_InputToForgetWeights",
1404 m_InputParams.m_InputToForgetWeights, params.m_InputToForgetWeights);
1405 VerifyConstTensors("m_InputToCellWeights",
1406 m_InputParams.m_InputToCellWeights, params.m_InputToCellWeights);
1407 VerifyConstTensors("m_InputToOutputWeights",
1408 m_InputParams.m_InputToOutputWeights, params.m_InputToOutputWeights);
1409 VerifyConstTensors("m_RecurrentToInputWeights",
1410 m_InputParams.m_RecurrentToInputWeights, params.m_RecurrentToInputWeights);
1411 VerifyConstTensors("m_RecurrentToForgetWeights",
1412 m_InputParams.m_RecurrentToForgetWeights, params.m_RecurrentToForgetWeights);
1413 VerifyConstTensors("m_RecurrentToCellWeights",
1414 m_InputParams.m_RecurrentToCellWeights, params.m_RecurrentToCellWeights);
1415 VerifyConstTensors("m_RecurrentToOutputWeights",
1416 m_InputParams.m_RecurrentToOutputWeights, params.m_RecurrentToOutputWeights);
1417 VerifyConstTensors("m_InputGateBias",
1418 m_InputParams.m_InputGateBias, params.m_InputGateBias);
1419 VerifyConstTensors("m_ForgetGateBias",
1420 m_InputParams.m_ForgetGateBias, params.m_ForgetGateBias);
1421 VerifyConstTensors("m_CellBias",
1422 m_InputParams.m_CellBias, params.m_CellBias);
1423 VerifyConstTensors("m_OutputGateBias",
1424 m_InputParams.m_OutputGateBias, params.m_OutputGateBias);
1425 }
1426
1427private:
1428 armnn::QuantizedLstmInputParams m_InputParams;
1429};
1430
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]1431TEST_CASE("SerializeDeserializeQuantizedLstm")
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]1432{
1433 const uint32_t batchSize = 1;
1434 const uint32_t inputSize = 2;
1435 const uint32_t numUnits = 4;
1436 const uint32_t outputSize = numUnits;
1437
1438 // Scale/Offset for input/output, cellState In/Out, weights, bias
1439 float inputOutputScale = 0.0078125f;
1440 int32_t inputOutputOffset = 128;
1441
1442 float cellStateScale = 0.00048828125f;
1443 int32_t cellStateOffset = 0;
1444
1445 float weightsScale = 0.00408021f;
1446 int32_t weightsOffset = 100;
1447
1448 float biasScale = 3.1876640625e-05f;
1449 int32_t biasOffset = 0;
1450
1451 // The shape of weight data is {outputSize, inputSize} = {4, 2}
1452 armnn::TensorShape inputToInputWeightsShape = {4, 2};
1453 std::vector<uint8_t> inputToInputWeightsData = {1, 2, 3, 4, 5, 6, 7, 8};
1454 armnn::TensorInfo inputToInputWeightsInfo(inputToInputWeightsShape,
1455 armnn::DataType::QAsymmU8,
1456 weightsScale,
1457 weightsOffset);
1458 armnn::ConstTensor inputToInputWeights(inputToInputWeightsInfo, inputToInputWeightsData);
1459
1460 armnn::TensorShape inputToForgetWeightsShape = {4, 2};
1461 std::vector<uint8_t> inputToForgetWeightsData = {1, 2, 3, 4, 5, 6, 7, 8};
1462 armnn::TensorInfo inputToForgetWeightsInfo(inputToForgetWeightsShape,
1463 armnn::DataType::QAsymmU8,
1464 weightsScale,
1465 weightsOffset);
1466 armnn::ConstTensor inputToForgetWeights(inputToForgetWeightsInfo, inputToForgetWeightsData);
1467
1468 armnn::TensorShape inputToCellWeightsShape = {4, 2};
1469 std::vector<uint8_t> inputToCellWeightsData = {1, 2, 3, 4, 5, 6, 7, 8};
1470 armnn::TensorInfo inputToCellWeightsInfo(inputToCellWeightsShape,
1471 armnn::DataType::QAsymmU8,
1472 weightsScale,
1473 weightsOffset);
1474 armnn::ConstTensor inputToCellWeights(inputToCellWeightsInfo, inputToCellWeightsData);
1475
1476 armnn::TensorShape inputToOutputWeightsShape = {4, 2};
1477 std::vector<uint8_t> inputToOutputWeightsData = {1, 2, 3, 4, 5, 6, 7, 8};
1478 armnn::TensorInfo inputToOutputWeightsInfo(inputToOutputWeightsShape,
1479 armnn::DataType::QAsymmU8,
1480 weightsScale,
1481 weightsOffset);
1482 armnn::ConstTensor inputToOutputWeights(inputToOutputWeightsInfo, inputToOutputWeightsData);
1483
1484 // The shape of recurrent weight data is {outputSize, outputSize} = {4, 4}
1485 armnn::TensorShape recurrentToInputWeightsShape = {4, 4};
1486 std::vector<uint8_t> recurrentToInputWeightsData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
1487 armnn::TensorInfo recurrentToInputWeightsInfo(recurrentToInputWeightsShape,
1488 armnn::DataType::QAsymmU8,
1489 weightsScale,
1490 weightsOffset);
1491 armnn::ConstTensor recurrentToInputWeights(recurrentToInputWeightsInfo, recurrentToInputWeightsData);
1492
1493 armnn::TensorShape recurrentToForgetWeightsShape = {4, 4};
1494 std::vector<uint8_t> recurrentToForgetWeightsData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
1495 armnn::TensorInfo recurrentToForgetWeightsInfo(recurrentToForgetWeightsShape,
1496 armnn::DataType::QAsymmU8,
1497 weightsScale,
1498 weightsOffset);
1499 armnn::ConstTensor recurrentToForgetWeights(recurrentToForgetWeightsInfo, recurrentToForgetWeightsData);
1500
1501 armnn::TensorShape recurrentToCellWeightsShape = {4, 4};
1502 std::vector<uint8_t> recurrentToCellWeightsData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
1503 armnn::TensorInfo recurrentToCellWeightsInfo(recurrentToCellWeightsShape,
1504 armnn::DataType::QAsymmU8,
1505 weightsScale,
1506 weightsOffset);
1507 armnn::ConstTensor recurrentToCellWeights(recurrentToCellWeightsInfo, recurrentToCellWeightsData);
1508
1509 armnn::TensorShape recurrentToOutputWeightsShape = {4, 4};
1510 std::vector<uint8_t> recurrentToOutputWeightsData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
1511 armnn::TensorInfo recurrentToOutputWeightsInfo(recurrentToOutputWeightsShape,
1512 armnn::DataType::QAsymmU8,
1513 weightsScale,
1514 weightsOffset);
1515 armnn::ConstTensor recurrentToOutputWeights(recurrentToOutputWeightsInfo, recurrentToOutputWeightsData);
1516
1517 // The shape of bias data is {outputSize} = {4}
1518 armnn::TensorShape inputGateBiasShape = {4};
1519 std::vector<int32_t> inputGateBiasData = {1, 2, 3, 4};
1520 armnn::TensorInfo inputGateBiasInfo(inputGateBiasShape,
1521 armnn::DataType::Signed32,
1522 biasScale,
1523 biasOffset);
1524 armnn::ConstTensor inputGateBias(inputGateBiasInfo, inputGateBiasData);
1525
1526 armnn::TensorShape forgetGateBiasShape = {4};
1527 std::vector<int32_t> forgetGateBiasData = {1, 2, 3, 4};
1528 armnn::TensorInfo forgetGateBiasInfo(forgetGateBiasShape,
1529 armnn::DataType::Signed32,
1530 biasScale,
1531 biasOffset);
1532 armnn::ConstTensor forgetGateBias(forgetGateBiasInfo, forgetGateBiasData);
1533
1534 armnn::TensorShape cellBiasShape = {4};
1535 std::vector<int32_t> cellBiasData = {1, 2, 3, 4};
1536 armnn::TensorInfo cellBiasInfo(cellBiasShape,
1537 armnn::DataType::Signed32,
1538 biasScale,
1539 biasOffset);
1540 armnn::ConstTensor cellBias(cellBiasInfo, cellBiasData);
1541
1542 armnn::TensorShape outputGateBiasShape = {4};
1543 std::vector<int32_t> outputGateBiasData = {1, 2, 3, 4};
1544 armnn::TensorInfo outputGateBiasInfo(outputGateBiasShape,
1545 armnn::DataType::Signed32,
1546 biasScale,
1547 biasOffset);
1548 armnn::ConstTensor outputGateBias(outputGateBiasInfo, outputGateBiasData);
1549
1550 armnn::QuantizedLstmInputParams params;
1551 params.m_InputToInputWeights = &inputToInputWeights;
1552 params.m_InputToForgetWeights = &inputToForgetWeights;
1553 params.m_InputToCellWeights = &inputToCellWeights;
1554 params.m_InputToOutputWeights = &inputToOutputWeights;
1555 params.m_RecurrentToInputWeights = &recurrentToInputWeights;
1556 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
1557 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
1558 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
1559 params.m_InputGateBias = &inputGateBias;
1560 params.m_ForgetGateBias = &forgetGateBias;
1561 params.m_CellBias = &cellBias;
1562 params.m_OutputGateBias = &outputGateBias;
1563
1564 armnn::INetworkPtr network = armnn::INetwork::Create();
1565 armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(0);
1566 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(1);
1567 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(2);
1568 const std::string layerName("QuantizedLstm");
1569 armnn::IConnectableLayer* const quantizedLstmLayer = network->AddQuantizedLstmLayer(params, layerName.c_str());
1570 armnn::IConnectableLayer* const cellStateOut = network->AddOutputLayer(0);
1571 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(1);
1572
1573 // Connect up
1574 armnn::TensorInfo inputTensorInfo({ batchSize, inputSize },
1575 armnn::DataType::QAsymmU8,
1576 inputOutputScale,
1577 inputOutputOffset);
1578 armnn::TensorInfo cellStateTensorInfo({ batchSize, numUnits },
1579 armnn::DataType::QSymmS16,
1580 cellStateScale,
1581 cellStateOffset);
1582 armnn::TensorInfo outputStateTensorInfo({ batchSize, outputSize },
1583 armnn::DataType::QAsymmU8,
1584 inputOutputScale,
1585 inputOutputOffset);
1586
1587 inputLayer->GetOutputSlot(0).Connect(quantizedLstmLayer->GetInputSlot(0));
1588 inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
1589
1590 cellStateIn->GetOutputSlot(0).Connect(quantizedLstmLayer->GetInputSlot(1));
1591 cellStateIn->GetOutputSlot(0).SetTensorInfo(cellStateTensorInfo);
1592
1593 outputStateIn->GetOutputSlot(0).Connect(quantizedLstmLayer->GetInputSlot(2));
1594 outputStateIn->GetOutputSlot(0).SetTensorInfo(outputStateTensorInfo);
1595
1596 quantizedLstmLayer->GetOutputSlot(0).Connect(cellStateOut->GetInputSlot(0));
1597 quantizedLstmLayer->GetOutputSlot(0).SetTensorInfo(cellStateTensorInfo);
1598
1599 quantizedLstmLayer->GetOutputSlot(1).Connect(outputLayer->GetInputSlot(0));
1600 quantizedLstmLayer->GetOutputSlot(1).SetTensorInfo(outputStateTensorInfo);
1601
1602 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]1603 CHECK(deserializedNetwork);
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]1604
1605 VerifyQuantizedLstmLayer checker(layerName,
1606 {inputTensorInfo, cellStateTensorInfo, outputStateTensorInfo},
1607 {cellStateTensorInfo, outputStateTensorInfo},
1608 params);
1609
1610 deserializedNetwork->ExecuteStrategy(checker);
1611}
1612
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]1613TEST_CASE("SerializeDeserializeQLstmBasic")
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]1614{
1615 armnn::QLstmDescriptor descriptor;
1616
1617 descriptor.m_CifgEnabled = true;
1618 descriptor.m_ProjectionEnabled = false;
1619 descriptor.m_PeepholeEnabled = false;
1620 descriptor.m_LayerNormEnabled = false;
1621
1622 descriptor.m_CellClip = 0.0f;
1623 descriptor.m_ProjectionClip = 0.0f;
1624
1625 descriptor.m_InputIntermediateScale = 0.00001f;
1626 descriptor.m_ForgetIntermediateScale = 0.00001f;
1627 descriptor.m_CellIntermediateScale = 0.00001f;
1628 descriptor.m_OutputIntermediateScale = 0.00001f;
1629
1630 descriptor.m_HiddenStateScale = 0.07f;
1631 descriptor.m_HiddenStateZeroPoint = 0;
1632
1633 const unsigned int numBatches = 2;
1634 const unsigned int inputSize = 5;
1635 const unsigned int outputSize = 4;
1636 const unsigned int numUnits = 4;
1637
1638 // Scale/Offset quantization info
1639 float inputScale = 0.0078f;
1640 int32_t inputOffset = 0;
1641
1642 float outputScale = 0.0078f;
1643 int32_t outputOffset = 0;
1644
1645 float cellStateScale = 3.5002e-05f;
1646 int32_t cellStateOffset = 0;
1647
1648 float weightsScale = 0.007f;
1649 int32_t weightsOffset = 0;
1650
1651 float biasScale = 3.5002e-05f / 1024;
1652 int32_t biasOffset = 0;
1653
1654 // Weights and bias tensor and quantization info
1655 armnn::TensorInfo inputWeightsInfo({numUnits, inputSize},
1656 armnn::DataType::QSymmS8,
1657 weightsScale,
1658 weightsOffset);
1659
1660 armnn::TensorInfo recurrentWeightsInfo({numUnits, outputSize},
1661 armnn::DataType::QSymmS8,
1662 weightsScale,
1663 weightsOffset);
1664
1665 armnn::TensorInfo biasInfo({numUnits}, armnn::DataType::Signed32, biasScale, biasOffset);
1666
1667 std::vector<int8_t> inputToForgetWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
1668 std::vector<int8_t> inputToCellWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
1669 std::vector<int8_t> inputToOutputWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
1670
1671 armnn::ConstTensor inputToForgetWeights(inputWeightsInfo, inputToForgetWeightsData);
1672 armnn::ConstTensor inputToCellWeights(inputWeightsInfo, inputToCellWeightsData);
1673 armnn::ConstTensor inputToOutputWeights(inputWeightsInfo, inputToOutputWeightsData);
1674
1675 std::vector<int8_t> recurrentToForgetWeightsData =
1676 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
1677 std::vector<int8_t> recurrentToCellWeightsData =
1678 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
1679 std::vector<int8_t> recurrentToOutputWeightsData =
1680 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
1681
1682 armnn::ConstTensor recurrentToForgetWeights(recurrentWeightsInfo, recurrentToForgetWeightsData);
1683 armnn::ConstTensor recurrentToCellWeights(recurrentWeightsInfo, recurrentToCellWeightsData);
1684 armnn::ConstTensor recurrentToOutputWeights(recurrentWeightsInfo, recurrentToOutputWeightsData);
1685
1686 std::vector<int32_t> forgetGateBiasData(numUnits, 1);
1687 std::vector<int32_t> cellBiasData(numUnits, 0);
1688 std::vector<int32_t> outputGateBiasData(numUnits, 0);
1689
1690 armnn::ConstTensor forgetGateBias(biasInfo, forgetGateBiasData);
1691 armnn::ConstTensor cellBias(biasInfo, cellBiasData);
1692 armnn::ConstTensor outputGateBias(biasInfo, outputGateBiasData);
1693
1694 // Set up params
1695 armnn::LstmInputParams params;
1696 params.m_InputToForgetWeights = &inputToForgetWeights;
1697 params.m_InputToCellWeights = &inputToCellWeights;
1698 params.m_InputToOutputWeights = &inputToOutputWeights;
1699
1700 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
1701 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
1702 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
1703
1704 params.m_ForgetGateBias = &forgetGateBias;
1705 params.m_CellBias = &cellBias;
1706 params.m_OutputGateBias = &outputGateBias;
1707
1708 // Create network
1709 armnn::INetworkPtr network = armnn::INetwork::Create();
1710 const std::string layerName("qLstm");
1711
1712 armnn::IConnectableLayer* const input = network->AddInputLayer(0);
1713 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(1);
1714 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(2);
1715
1716 armnn::IConnectableLayer* const qLstmLayer = network->AddQLstmLayer(descriptor, params, layerName.c_str());
1717
1718 armnn::IConnectableLayer* const outputStateOut = network->AddOutputLayer(0);
1719 armnn::IConnectableLayer* const cellStateOut = network->AddOutputLayer(1);
1720 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(2);
1721
1722 // Input/Output tensor info
1723 armnn::TensorInfo inputInfo({numBatches , inputSize},
1724 armnn::DataType::QAsymmS8,
1725 inputScale,
1726 inputOffset);
1727
1728 armnn::TensorInfo cellStateInfo({numBatches , numUnits},
1729 armnn::DataType::QSymmS16,
1730 cellStateScale,
1731 cellStateOffset);
1732
1733 armnn::TensorInfo outputStateInfo({numBatches , outputSize},
1734 armnn::DataType::QAsymmS8,
1735 outputScale,
1736 outputOffset);
1737
1738 // Connect input/output slots
1739 input->GetOutputSlot(0).Connect(qLstmLayer->GetInputSlot(0));
1740 input->GetOutputSlot(0).SetTensorInfo(inputInfo);
1741
1742 outputStateIn->GetOutputSlot(0).Connect(qLstmLayer->GetInputSlot(1));
1743 outputStateIn->GetOutputSlot(0).SetTensorInfo(cellStateInfo);
1744
1745 cellStateIn->GetOutputSlot(0).Connect(qLstmLayer->GetInputSlot(2));
1746 cellStateIn->GetOutputSlot(0).SetTensorInfo(outputStateInfo);
1747
1748 qLstmLayer->GetOutputSlot(0).Connect(outputStateOut->GetInputSlot(0));
1749 qLstmLayer->GetOutputSlot(0).SetTensorInfo(outputStateInfo);
1750
1751 qLstmLayer->GetOutputSlot(1).Connect(cellStateOut->GetInputSlot(0));
1752 qLstmLayer->GetOutputSlot(1).SetTensorInfo(cellStateInfo);
1753
1754 qLstmLayer->GetOutputSlot(2).Connect(outputLayer->GetInputSlot(0));
1755 qLstmLayer->GetOutputSlot(2).SetTensorInfo(outputStateInfo);
1756
1757 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]1758 CHECK(deserializedNetwork);
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]1759
1760 VerifyLstmLayer<armnn::QLstmDescriptor> checker(
1761 layerName,
1762 {inputInfo, cellStateInfo, outputStateInfo},
1763 {outputStateInfo, cellStateInfo, outputStateInfo},
1764 descriptor,
1765 params);
1766
1767 deserializedNetwork->ExecuteStrategy(checker);
1768}
1769
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]1770TEST_CASE("SerializeDeserializeQLstmCifgLayerNorm")
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]1771{
1772 armnn::QLstmDescriptor descriptor;
1773
1774 // CIFG params are used when CIFG is disabled
1775 descriptor.m_CifgEnabled = true;
1776 descriptor.m_ProjectionEnabled = false;
1777 descriptor.m_PeepholeEnabled = false;
1778 descriptor.m_LayerNormEnabled = true;
1779
1780 descriptor.m_CellClip = 0.0f;
1781 descriptor.m_ProjectionClip = 0.0f;
1782
1783 descriptor.m_InputIntermediateScale = 0.00001f;
1784 descriptor.m_ForgetIntermediateScale = 0.00001f;
1785 descriptor.m_CellIntermediateScale = 0.00001f;
1786 descriptor.m_OutputIntermediateScale = 0.00001f;
1787
1788 descriptor.m_HiddenStateScale = 0.07f;
1789 descriptor.m_HiddenStateZeroPoint = 0;
1790
1791 const unsigned int numBatches = 2;
1792 const unsigned int inputSize = 5;
1793 const unsigned int outputSize = 4;
1794 const unsigned int numUnits = 4;
1795
1796 // Scale/Offset quantization info
1797 float inputScale = 0.0078f;
1798 int32_t inputOffset = 0;
1799
1800 float outputScale = 0.0078f;
1801 int32_t outputOffset = 0;
1802
1803 float cellStateScale = 3.5002e-05f;
1804 int32_t cellStateOffset = 0;
1805
1806 float weightsScale = 0.007f;
1807 int32_t weightsOffset = 0;
1808
1809 float layerNormScale = 3.5002e-05f;
1810 int32_t layerNormOffset = 0;
1811
1812 float biasScale = layerNormScale / 1024;
1813 int32_t biasOffset = 0;
1814
1815 // Weights and bias tensor and quantization info
1816 armnn::TensorInfo inputWeightsInfo({numUnits, inputSize},
1817 armnn::DataType::QSymmS8,
1818 weightsScale,
1819 weightsOffset);
1820
1821 armnn::TensorInfo recurrentWeightsInfo({numUnits, outputSize},
1822 armnn::DataType::QSymmS8,
1823 weightsScale,
1824 weightsOffset);
1825
1826 armnn::TensorInfo biasInfo({numUnits},
1827 armnn::DataType::Signed32,
1828 biasScale,
1829 biasOffset);
1830
1831 armnn::TensorInfo layerNormWeightsInfo({numUnits},
1832 armnn::DataType::QSymmS16,
1833 layerNormScale,
1834 layerNormOffset);
1835
1836 // Mandatory params
1837 std::vector<int8_t> inputToForgetWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
1838 std::vector<int8_t> inputToCellWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
1839 std::vector<int8_t> inputToOutputWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
1840
1841 armnn::ConstTensor inputToForgetWeights(inputWeightsInfo, inputToForgetWeightsData);
1842 armnn::ConstTensor inputToCellWeights(inputWeightsInfo, inputToCellWeightsData);
1843 armnn::ConstTensor inputToOutputWeights(inputWeightsInfo, inputToOutputWeightsData);
1844
1845 std::vector<int8_t> recurrentToForgetWeightsData =
1846 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
1847 std::vector<int8_t> recurrentToCellWeightsData =
1848 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
1849 std::vector<int8_t> recurrentToOutputWeightsData =
1850 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
1851
1852 armnn::ConstTensor recurrentToForgetWeights(recurrentWeightsInfo, recurrentToForgetWeightsData);
1853 armnn::ConstTensor recurrentToCellWeights(recurrentWeightsInfo, recurrentToCellWeightsData);
1854 armnn::ConstTensor recurrentToOutputWeights(recurrentWeightsInfo, recurrentToOutputWeightsData);
1855
1856 std::vector<int32_t> forgetGateBiasData(numUnits, 1);
1857 std::vector<int32_t> cellBiasData(numUnits, 0);
1858 std::vector<int32_t> outputGateBiasData(numUnits, 0);
1859
1860 armnn::ConstTensor forgetGateBias(biasInfo, forgetGateBiasData);
1861 armnn::ConstTensor cellBias(biasInfo, cellBiasData);
1862 armnn::ConstTensor outputGateBias(biasInfo, outputGateBiasData);
1863
1864 // Layer Norm
1865 std::vector<int16_t> forgetLayerNormWeightsData =
1866 GenerateRandomData<int16_t>(layerNormWeightsInfo.GetNumElements());
1867 std::vector<int16_t> cellLayerNormWeightsData =
1868 GenerateRandomData<int16_t>(layerNormWeightsInfo.GetNumElements());
1869 std::vector<int16_t> outputLayerNormWeightsData =
1870 GenerateRandomData<int16_t>(layerNormWeightsInfo.GetNumElements());
1871
1872 armnn::ConstTensor forgetLayerNormWeights(layerNormWeightsInfo, forgetLayerNormWeightsData);
1873 armnn::ConstTensor cellLayerNormWeights(layerNormWeightsInfo, cellLayerNormWeightsData);
1874 armnn::ConstTensor outputLayerNormWeights(layerNormWeightsInfo, outputLayerNormWeightsData);
1875
1876 // Set up params
1877 armnn::LstmInputParams params;
1878
1879 // Mandatory params
1880 params.m_InputToForgetWeights = &inputToForgetWeights;
1881 params.m_InputToCellWeights = &inputToCellWeights;
1882 params.m_InputToOutputWeights = &inputToOutputWeights;
1883
1884 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
1885 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
1886 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
1887
1888 params.m_ForgetGateBias = &forgetGateBias;
1889 params.m_CellBias = &cellBias;
1890 params.m_OutputGateBias = &outputGateBias;
1891
1892 // Layer Norm
1893 params.m_ForgetLayerNormWeights = &forgetLayerNormWeights;
1894 params.m_CellLayerNormWeights = &cellLayerNormWeights;
1895 params.m_OutputLayerNormWeights = &outputLayerNormWeights;
1896
1897 // Create network
1898 armnn::INetworkPtr network = armnn::INetwork::Create();
1899 const std::string layerName("qLstm");
1900
1901 armnn::IConnectableLayer* const input = network->AddInputLayer(0);
1902 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(1);
1903 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(2);
1904
1905 armnn::IConnectableLayer* const qLstmLayer = network->AddQLstmLayer(descriptor, params, layerName.c_str());
1906
1907 armnn::IConnectableLayer* const outputStateOut = network->AddOutputLayer(0);
1908 armnn::IConnectableLayer* const cellStateOut = network->AddOutputLayer(1);
1909 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(2);
1910
1911 // Input/Output tensor info
1912 armnn::TensorInfo inputInfo({numBatches , inputSize},
1913 armnn::DataType::QAsymmS8,
1914 inputScale,
1915 inputOffset);
1916
1917 armnn::TensorInfo cellStateInfo({numBatches , numUnits},
1918 armnn::DataType::QSymmS16,
1919 cellStateScale,
1920 cellStateOffset);
1921
1922 armnn::TensorInfo outputStateInfo({numBatches , outputSize},
1923 armnn::DataType::QAsymmS8,
1924 outputScale,
1925 outputOffset);
1926
1927 // Connect input/output slots
1928 input->GetOutputSlot(0).Connect(qLstmLayer->GetInputSlot(0));
1929 input->GetOutputSlot(0).SetTensorInfo(inputInfo);
1930
1931 outputStateIn->GetOutputSlot(0).Connect(qLstmLayer->GetInputSlot(1));
1932 outputStateIn->GetOutputSlot(0).SetTensorInfo(cellStateInfo);
1933
1934 cellStateIn->GetOutputSlot(0).Connect(qLstmLayer->GetInputSlot(2));
1935 cellStateIn->GetOutputSlot(0).SetTensorInfo(outputStateInfo);
1936
1937 qLstmLayer->GetOutputSlot(0).Connect(outputStateOut->GetInputSlot(0));
1938 qLstmLayer->GetOutputSlot(0).SetTensorInfo(outputStateInfo);
1939
1940 qLstmLayer->GetOutputSlot(1).Connect(cellStateOut->GetInputSlot(0));
1941 qLstmLayer->GetOutputSlot(1).SetTensorInfo(cellStateInfo);
1942
1943 qLstmLayer->GetOutputSlot(2).Connect(outputLayer->GetInputSlot(0));
1944 qLstmLayer->GetOutputSlot(2).SetTensorInfo(outputStateInfo);
1945
1946 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]1947 CHECK(deserializedNetwork);
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]1948
1949 VerifyLstmLayer<armnn::QLstmDescriptor> checker(layerName,
1950 {inputInfo, cellStateInfo, outputStateInfo},
1951 {outputStateInfo, cellStateInfo, outputStateInfo},
1952 descriptor,
1953 params);
1954
1955 deserializedNetwork->ExecuteStrategy(checker);
1956}
1957
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]1958TEST_CASE("SerializeDeserializeQLstmAdvanced")
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]1959{
1960 armnn::QLstmDescriptor descriptor;
1961
1962 descriptor.m_CifgEnabled = false;
1963 descriptor.m_ProjectionEnabled = true;
1964 descriptor.m_PeepholeEnabled = true;
1965 descriptor.m_LayerNormEnabled = true;
1966
1967 descriptor.m_CellClip = 0.1f;
1968 descriptor.m_ProjectionClip = 0.1f;
1969
1970 descriptor.m_InputIntermediateScale = 0.00001f;
1971 descriptor.m_ForgetIntermediateScale = 0.00001f;
1972 descriptor.m_CellIntermediateScale = 0.00001f;
1973 descriptor.m_OutputIntermediateScale = 0.00001f;
1974
1975 descriptor.m_HiddenStateScale = 0.07f;
1976 descriptor.m_HiddenStateZeroPoint = 0;
1977
1978 const unsigned int numBatches = 2;
1979 const unsigned int inputSize = 5;
1980 const unsigned int outputSize = 4;
1981 const unsigned int numUnits = 4;
1982
1983 // Scale/Offset quantization info
1984 float inputScale = 0.0078f;
1985 int32_t inputOffset = 0;
1986
1987 float outputScale = 0.0078f;
1988 int32_t outputOffset = 0;
1989
1990 float cellStateScale = 3.5002e-05f;
1991 int32_t cellStateOffset = 0;
1992
1993 float weightsScale = 0.007f;
1994 int32_t weightsOffset = 0;
1995
1996 float layerNormScale = 3.5002e-05f;
1997 int32_t layerNormOffset = 0;
1998
1999 float biasScale = layerNormScale / 1024;
2000 int32_t biasOffset = 0;
2001
2002 // Weights and bias tensor and quantization info
2003 armnn::TensorInfo inputWeightsInfo({numUnits, inputSize},
2004 armnn::DataType::QSymmS8,
2005 weightsScale,
2006 weightsOffset);
2007
2008 armnn::TensorInfo recurrentWeightsInfo({numUnits, outputSize},
2009 armnn::DataType::QSymmS8,
2010 weightsScale,
2011 weightsOffset);
2012
2013 armnn::TensorInfo biasInfo({numUnits},
2014 armnn::DataType::Signed32,
2015 biasScale,
2016 biasOffset);
2017
2018 armnn::TensorInfo peepholeWeightsInfo({numUnits},
2019 armnn::DataType::QSymmS16,
2020 weightsScale,
2021 weightsOffset);
2022
2023 armnn::TensorInfo layerNormWeightsInfo({numUnits},
2024 armnn::DataType::QSymmS16,
2025 layerNormScale,
2026 layerNormOffset);
2027
2028 armnn::TensorInfo projectionWeightsInfo({outputSize, numUnits},
2029 armnn::DataType::QSymmS8,
2030 weightsScale,
2031 weightsOffset);
2032
2033 // Mandatory params
2034 std::vector<int8_t> inputToForgetWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
2035 std::vector<int8_t> inputToCellWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
2036 std::vector<int8_t> inputToOutputWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
2037
2038 armnn::ConstTensor inputToForgetWeights(inputWeightsInfo, inputToForgetWeightsData);
2039 armnn::ConstTensor inputToCellWeights(inputWeightsInfo, inputToCellWeightsData);
2040 armnn::ConstTensor inputToOutputWeights(inputWeightsInfo, inputToOutputWeightsData);
2041
2042 std::vector<int8_t> recurrentToForgetWeightsData =
2043 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
2044 std::vector<int8_t> recurrentToCellWeightsData =
2045 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
2046 std::vector<int8_t> recurrentToOutputWeightsData =
2047 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
2048
2049 armnn::ConstTensor recurrentToForgetWeights(recurrentWeightsInfo, recurrentToForgetWeightsData);
2050 armnn::ConstTensor recurrentToCellWeights(recurrentWeightsInfo, recurrentToCellWeightsData);
2051 armnn::ConstTensor recurrentToOutputWeights(recurrentWeightsInfo, recurrentToOutputWeightsData);
2052
2053 std::vector<int32_t> forgetGateBiasData(numUnits, 1);
2054 std::vector<int32_t> cellBiasData(numUnits, 0);
2055 std::vector<int32_t> outputGateBiasData(numUnits, 0);
2056
2057 armnn::ConstTensor forgetGateBias(biasInfo, forgetGateBiasData);
2058 armnn::ConstTensor cellBias(biasInfo, cellBiasData);
2059 armnn::ConstTensor outputGateBias(biasInfo, outputGateBiasData);
2060
2061 // CIFG
2062 std::vector<int8_t> inputToInputWeightsData = GenerateRandomData<int8_t>(inputWeightsInfo.GetNumElements());
2063 std::vector<int8_t> recurrentToInputWeightsData =
2064 GenerateRandomData<int8_t>(recurrentWeightsInfo.GetNumElements());
2065 std::vector<int32_t> inputGateBiasData(numUnits, 1);
2066
2067 armnn::ConstTensor inputToInputWeights(inputWeightsInfo, inputToInputWeightsData);
2068 armnn::ConstTensor recurrentToInputWeights(recurrentWeightsInfo, recurrentToInputWeightsData);
2069 armnn::ConstTensor inputGateBias(biasInfo, inputGateBiasData);
2070
2071 // Peephole
2072 std::vector<int16_t> cellToInputWeightsData = GenerateRandomData<int16_t>(peepholeWeightsInfo.GetNumElements());
2073 std::vector<int16_t> cellToForgetWeightsData = GenerateRandomData<int16_t>(peepholeWeightsInfo.GetNumElements());
2074 std::vector<int16_t> cellToOutputWeightsData = GenerateRandomData<int16_t>(peepholeWeightsInfo.GetNumElements());
2075
2076 armnn::ConstTensor cellToInputWeights(peepholeWeightsInfo, cellToInputWeightsData);
2077 armnn::ConstTensor cellToForgetWeights(peepholeWeightsInfo, cellToForgetWeightsData);
2078 armnn::ConstTensor cellToOutputWeights(peepholeWeightsInfo, cellToOutputWeightsData);
2079
2080 // Projection
2081 std::vector<int8_t> projectionWeightsData = GenerateRandomData<int8_t>(projectionWeightsInfo.GetNumElements());
2082 std::vector<int32_t> projectionBiasData(outputSize, 1);
2083
2084 armnn::ConstTensor projectionWeights(projectionWeightsInfo, projectionWeightsData);
2085 armnn::ConstTensor projectionBias(biasInfo, projectionBiasData);
2086
2087 // Layer Norm
2088 std::vector<int16_t> inputLayerNormWeightsData =
2089 GenerateRandomData<int16_t>(layerNormWeightsInfo.GetNumElements());
2090 std::vector<int16_t> forgetLayerNormWeightsData =
2091 GenerateRandomData<int16_t>(layerNormWeightsInfo.GetNumElements());
2092 std::vector<int16_t> cellLayerNormWeightsData =
2093 GenerateRandomData<int16_t>(layerNormWeightsInfo.GetNumElements());
2094 std::vector<int16_t> outputLayerNormWeightsData =
2095 GenerateRandomData<int16_t>(layerNormWeightsInfo.GetNumElements());
2096
2097 armnn::ConstTensor inputLayerNormWeights(layerNormWeightsInfo, inputLayerNormWeightsData);
2098 armnn::ConstTensor forgetLayerNormWeights(layerNormWeightsInfo, forgetLayerNormWeightsData);
2099 armnn::ConstTensor cellLayerNormWeights(layerNormWeightsInfo, cellLayerNormWeightsData);
2100 armnn::ConstTensor outputLayerNormWeights(layerNormWeightsInfo, outputLayerNormWeightsData);
2101
2102 // Set up params
2103 armnn::LstmInputParams params;
2104
2105 // Mandatory params
2106 params.m_InputToForgetWeights = &inputToForgetWeights;
2107 params.m_InputToCellWeights = &inputToCellWeights;
2108 params.m_InputToOutputWeights = &inputToOutputWeights;
2109
2110 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
2111 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
2112 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
2113
2114 params.m_ForgetGateBias = &forgetGateBias;
2115 params.m_CellBias = &cellBias;
2116 params.m_OutputGateBias = &outputGateBias;
2117
2118 // CIFG
2119 params.m_InputToInputWeights = &inputToInputWeights;
2120 params.m_RecurrentToInputWeights = &recurrentToInputWeights;
2121 params.m_InputGateBias = &inputGateBias;
2122
2123 // Peephole
2124 params.m_CellToInputWeights = &cellToInputWeights;
2125 params.m_CellToForgetWeights = &cellToForgetWeights;
2126 params.m_CellToOutputWeights = &cellToOutputWeights;
2127
2128 // Projection
2129 params.m_ProjectionWeights = &projectionWeights;
2130 params.m_ProjectionBias = &projectionBias;
2131
2132 // Layer Norm
2133 params.m_InputLayerNormWeights = &inputLayerNormWeights;
2134 params.m_ForgetLayerNormWeights = &forgetLayerNormWeights;
2135 params.m_CellLayerNormWeights = &cellLayerNormWeights;
2136 params.m_OutputLayerNormWeights = &outputLayerNormWeights;
2137
2138 // Create network
2139 armnn::INetworkPtr network = armnn::INetwork::Create();
2140 const std::string layerName("qLstm");
2141
2142 armnn::IConnectableLayer* const input = network->AddInputLayer(0);
2143 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(1);
2144 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(2);
2145
2146 armnn::IConnectableLayer* const qLstmLayer = network->AddQLstmLayer(descriptor, params, layerName.c_str());
2147
2148 armnn::IConnectableLayer* const outputStateOut = network->AddOutputLayer(0);
2149 armnn::IConnectableLayer* const cellStateOut = network->AddOutputLayer(1);
2150 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(2);
2151
2152 // Input/Output tensor info
2153 armnn::TensorInfo inputInfo({numBatches , inputSize},
2154 armnn::DataType::QAsymmS8,
2155 inputScale,
2156 inputOffset);
2157
2158 armnn::TensorInfo cellStateInfo({numBatches , numUnits},
2159 armnn::DataType::QSymmS16,
2160 cellStateScale,
2161 cellStateOffset);
2162
2163 armnn::TensorInfo outputStateInfo({numBatches , outputSize},
2164 armnn::DataType::QAsymmS8,
2165 outputScale,
2166 outputOffset);
2167
2168 // Connect input/output slots
2169 input->GetOutputSlot(0).Connect(qLstmLayer->GetInputSlot(0));
2170 input->GetOutputSlot(0).SetTensorInfo(inputInfo);
2171
2172 outputStateIn->GetOutputSlot(0).Connect(qLstmLayer->GetInputSlot(1));
2173 outputStateIn->GetOutputSlot(0).SetTensorInfo(cellStateInfo);
2174
2175 cellStateIn->GetOutputSlot(0).Connect(qLstmLayer->GetInputSlot(2));
2176 cellStateIn->GetOutputSlot(0).SetTensorInfo(outputStateInfo);
2177
2178 qLstmLayer->GetOutputSlot(0).Connect(outputStateOut->GetInputSlot(0));
2179 qLstmLayer->GetOutputSlot(0).SetTensorInfo(outputStateInfo);
2180
2181 qLstmLayer->GetOutputSlot(1).Connect(cellStateOut->GetInputSlot(0));
2182 qLstmLayer->GetOutputSlot(1).SetTensorInfo(cellStateInfo);
2183
2184 qLstmLayer->GetOutputSlot(2).Connect(outputLayer->GetInputSlot(0));
2185 qLstmLayer->GetOutputSlot(2).SetTensorInfo(outputStateInfo);
2186
2187 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]2188 CHECK(deserializedNetwork);
Finn Williamsb454c5c2021-02-09 15:56:23 +0000[diff] [blame]2189
2190 VerifyLstmLayer<armnn::QLstmDescriptor> checker(layerName,
2191 {inputInfo, cellStateInfo, outputStateInfo},
2192 {outputStateInfo, cellStateInfo, outputStateInfo},
2193 descriptor,
2194 params);
2195
2196 deserializedNetwork->ExecuteStrategy(checker);
2197}
2198
Narumol Prangnawarata0162e12021-07-23 14:47:49 +0100[diff] [blame]2199TEST_CASE("SerializeDeserializeUnidirectionalSequenceLstmCifgPeepholeNoProjection")
2200{
2201 armnn::UnidirectionalSequenceLstmDescriptor descriptor;
2202 descriptor.m_ActivationFunc = 4;
2203 descriptor.m_ClippingThresProj = 0.0f;
2204 descriptor.m_ClippingThresCell = 0.0f;
2205 descriptor.m_CifgEnabled = true; // if this is true then we DON'T need to set the OptCifgParams
2206 descriptor.m_ProjectionEnabled = false;
2207 descriptor.m_PeepholeEnabled = true;
2208 descriptor.m_TimeMajor = false;
2209
2210 const uint32_t batchSize = 1;
2211 const uint32_t timeSize = 2;
2212 const uint32_t inputSize = 2;
2213 const uint32_t numUnits = 4;
2214 const uint32_t outputSize = numUnits;
2215
2216 armnn::TensorInfo inputWeightsInfo1({numUnits, inputSize}, armnn::DataType::Float32);
2217 std::vector<float> inputToForgetWeightsData = GenerateRandomData<float>(inputWeightsInfo1.GetNumElements());
2218 armnn::ConstTensor inputToForgetWeights(inputWeightsInfo1, inputToForgetWeightsData);
2219
2220 std::vector<float> inputToCellWeightsData = GenerateRandomData<float>(inputWeightsInfo1.GetNumElements());
2221 armnn::ConstTensor inputToCellWeights(inputWeightsInfo1, inputToCellWeightsData);
2222
2223 std::vector<float> inputToOutputWeightsData = GenerateRandomData<float>(inputWeightsInfo1.GetNumElements());
2224 armnn::ConstTensor inputToOutputWeights(inputWeightsInfo1, inputToOutputWeightsData);
2225
2226 armnn::TensorInfo inputWeightsInfo2({numUnits, outputSize}, armnn::DataType::Float32);
2227 std::vector<float> recurrentToForgetWeightsData = GenerateRandomData<float>(inputWeightsInfo2.GetNumElements());
2228 armnn::ConstTensor recurrentToForgetWeights(inputWeightsInfo2, recurrentToForgetWeightsData);
2229
2230 std::vector<float> recurrentToCellWeightsData = GenerateRandomData<float>(inputWeightsInfo2.GetNumElements());
2231 armnn::ConstTensor recurrentToCellWeights(inputWeightsInfo2, recurrentToCellWeightsData);
2232
2233 std::vector<float> recurrentToOutputWeightsData = GenerateRandomData<float>(inputWeightsInfo2.GetNumElements());
2234 armnn::ConstTensor recurrentToOutputWeights(inputWeightsInfo2, recurrentToOutputWeightsData);
2235
2236 armnn::TensorInfo inputWeightsInfo3({numUnits}, armnn::DataType::Float32);
2237 std::vector<float> cellToForgetWeightsData = GenerateRandomData<float>(inputWeightsInfo3.GetNumElements());
2238 armnn::ConstTensor cellToForgetWeights(inputWeightsInfo3, cellToForgetWeightsData);
2239
2240 std::vector<float> cellToOutputWeightsData = GenerateRandomData<float>(inputWeightsInfo3.GetNumElements());
2241 armnn::ConstTensor cellToOutputWeights(inputWeightsInfo3, cellToOutputWeightsData);
2242
2243 std::vector<float> forgetGateBiasData(numUnits, 1.0f);
2244 armnn::ConstTensor forgetGateBias(inputWeightsInfo3, forgetGateBiasData);
2245
2246 std::vector<float> cellBiasData(numUnits, 0.0f);
2247 armnn::ConstTensor cellBias(inputWeightsInfo3, cellBiasData);
2248
2249 std::vector<float> outputGateBiasData(numUnits, 0.0f);
2250 armnn::ConstTensor outputGateBias(inputWeightsInfo3, outputGateBiasData);
2251
2252 armnn::LstmInputParams params;
2253 params.m_InputToForgetWeights = &inputToForgetWeights;
2254 params.m_InputToCellWeights = &inputToCellWeights;
2255 params.m_InputToOutputWeights = &inputToOutputWeights;
2256 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
2257 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
2258 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
2259 params.m_ForgetGateBias = &forgetGateBias;
2260 params.m_CellBias = &cellBias;
2261 params.m_OutputGateBias = &outputGateBias;
2262 params.m_CellToForgetWeights = &cellToForgetWeights;
2263 params.m_CellToOutputWeights = &cellToOutputWeights;
2264
2265 armnn::INetworkPtr network = armnn::INetwork::Create();
2266 armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(0);
2267 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(1);
2268 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(2);
2269 const std::string layerName("UnidirectionalSequenceLstm");
2270 armnn::IConnectableLayer* const unidirectionalSequenceLstmLayer =
2271 network->AddUnidirectionalSequenceLstmLayer(descriptor, params, layerName.c_str());
2272 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(0);
2273
2274 // connect up
2275 armnn::TensorInfo inputTensorInfo({ batchSize, timeSize, inputSize }, armnn::DataType::Float32);
2276 armnn::TensorInfo cellStateTensorInfo({ batchSize, numUnits}, armnn::DataType::Float32);
2277 armnn::TensorInfo outputStateTensorInfo({ batchSize, outputSize }, armnn::DataType::Float32);
2278 armnn::TensorInfo outputTensorInfo({ batchSize, timeSize, outputSize }, armnn::DataType::Float32);
2279
2280 inputLayer->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(0));
2281 inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
2282
2283 outputStateIn->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(1));
2284 outputStateIn->GetOutputSlot(0).SetTensorInfo(outputStateTensorInfo);
2285
2286 cellStateIn->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(2));
2287 cellStateIn->GetOutputSlot(0).SetTensorInfo(cellStateTensorInfo);
2288
2289 unidirectionalSequenceLstmLayer->GetOutputSlot(0).Connect(outputLayer->GetInputSlot(0));
2290 unidirectionalSequenceLstmLayer->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
2291
2292 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
2293 CHECK(deserializedNetwork);
2294
2295 VerifyLstmLayer<armnn::UnidirectionalSequenceLstmDescriptor> checker(
2296 layerName,
2297 {inputTensorInfo, outputStateTensorInfo, cellStateTensorInfo},
2298 {outputTensorInfo},
2299 descriptor,
2300 params);
2301 deserializedNetwork->ExecuteStrategy(checker);
2302}
2303
2304TEST_CASE("SerializeDeserializeUnidirectionalSequenceLstmNoCifgWithPeepholeAndProjection")
2305{
2306 armnn::UnidirectionalSequenceLstmDescriptor descriptor;
2307 descriptor.m_ActivationFunc = 4;
2308 descriptor.m_ClippingThresProj = 0.0f;
2309 descriptor.m_ClippingThresCell = 0.0f;
2310 descriptor.m_CifgEnabled = false; // if this is true then we DON'T need to set the OptCifgParams
2311 descriptor.m_ProjectionEnabled = true;
2312 descriptor.m_PeepholeEnabled = true;
2313 descriptor.m_TimeMajor = false;
2314
2315 const uint32_t batchSize = 2;
2316 const uint32_t timeSize = 2;
2317 const uint32_t inputSize = 5;
2318 const uint32_t numUnits = 20;
2319 const uint32_t outputSize = 16;
2320
2321 armnn::TensorInfo tensorInfo20x5({numUnits, inputSize}, armnn::DataType::Float32);
2322 std::vector<float> inputToInputWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
2323 armnn::ConstTensor inputToInputWeights(tensorInfo20x5, inputToInputWeightsData);
2324
2325 std::vector<float> inputToForgetWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
2326 armnn::ConstTensor inputToForgetWeights(tensorInfo20x5, inputToForgetWeightsData);
2327
2328 std::vector<float> inputToCellWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
2329 armnn::ConstTensor inputToCellWeights(tensorInfo20x5, inputToCellWeightsData);
2330
2331 std::vector<float> inputToOutputWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
2332 armnn::ConstTensor inputToOutputWeights(tensorInfo20x5, inputToOutputWeightsData);
2333
2334 armnn::TensorInfo tensorInfo20({numUnits}, armnn::DataType::Float32);
2335 std::vector<float> inputGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2336 armnn::ConstTensor inputGateBias(tensorInfo20, inputGateBiasData);
2337
2338 std::vector<float> forgetGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2339 armnn::ConstTensor forgetGateBias(tensorInfo20, forgetGateBiasData);
2340
2341 std::vector<float> cellBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2342 armnn::ConstTensor cellBias(tensorInfo20, cellBiasData);
2343
2344 std::vector<float> outputGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2345 armnn::ConstTensor outputGateBias(tensorInfo20, outputGateBiasData);
2346
2347 armnn::TensorInfo tensorInfo20x16({numUnits, outputSize}, armnn::DataType::Float32);
2348 std::vector<float> recurrentToInputWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
2349 armnn::ConstTensor recurrentToInputWeights(tensorInfo20x16, recurrentToInputWeightsData);
2350
2351 std::vector<float> recurrentToForgetWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
2352 armnn::ConstTensor recurrentToForgetWeights(tensorInfo20x16, recurrentToForgetWeightsData);
2353
2354 std::vector<float> recurrentToCellWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
2355 armnn::ConstTensor recurrentToCellWeights(tensorInfo20x16, recurrentToCellWeightsData);
2356
2357 std::vector<float> recurrentToOutputWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
2358 armnn::ConstTensor recurrentToOutputWeights(tensorInfo20x16, recurrentToOutputWeightsData);
2359
2360 std::vector<float> cellToInputWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2361 armnn::ConstTensor cellToInputWeights(tensorInfo20, cellToInputWeightsData);
2362
2363 std::vector<float> cellToForgetWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2364 armnn::ConstTensor cellToForgetWeights(tensorInfo20, cellToForgetWeightsData);
2365
2366 std::vector<float> cellToOutputWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2367 armnn::ConstTensor cellToOutputWeights(tensorInfo20, cellToOutputWeightsData);
2368
2369 armnn::TensorInfo tensorInfo16x20({outputSize, numUnits}, armnn::DataType::Float32);
2370 std::vector<float> projectionWeightsData = GenerateRandomData<float>(tensorInfo16x20.GetNumElements());
2371 armnn::ConstTensor projectionWeights(tensorInfo16x20, projectionWeightsData);
2372
2373 armnn::TensorInfo tensorInfo16({outputSize}, armnn::DataType::Float32);
2374 std::vector<float> projectionBiasData(outputSize, 0.f);
2375 armnn::ConstTensor projectionBias(tensorInfo16, projectionBiasData);
2376
2377 armnn::LstmInputParams params;
2378 params.m_InputToForgetWeights = &inputToForgetWeights;
2379 params.m_InputToCellWeights = &inputToCellWeights;
2380 params.m_InputToOutputWeights = &inputToOutputWeights;
2381 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
2382 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
2383 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
2384 params.m_ForgetGateBias = &forgetGateBias;
2385 params.m_CellBias = &cellBias;
2386 params.m_OutputGateBias = &outputGateBias;
2387
2388 // additional params because: descriptor.m_CifgEnabled = false
2389 params.m_InputToInputWeights = &inputToInputWeights;
2390 params.m_RecurrentToInputWeights = &recurrentToInputWeights;
2391 params.m_CellToInputWeights = &cellToInputWeights;
2392 params.m_InputGateBias = &inputGateBias;
2393
2394 // additional params because: descriptor.m_ProjectionEnabled = true
2395 params.m_ProjectionWeights = &projectionWeights;
2396 params.m_ProjectionBias = &projectionBias;
2397
2398 // additional params because: descriptor.m_PeepholeEnabled = true
2399 params.m_CellToForgetWeights = &cellToForgetWeights;
2400 params.m_CellToOutputWeights = &cellToOutputWeights;
2401
2402 armnn::INetworkPtr network = armnn::INetwork::Create();
2403 armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(0);
2404 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(1);
2405 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(2);
2406 const std::string layerName("unidirectionalSequenceLstm");
2407 armnn::IConnectableLayer* const unidirectionalSequenceLstmLayer =
2408 network->AddUnidirectionalSequenceLstmLayer(descriptor, params, layerName.c_str());
2409 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(0);
2410
2411 // connect up
2412 armnn::TensorInfo inputTensorInfo({ batchSize, timeSize, inputSize }, armnn::DataType::Float32);
2413 armnn::TensorInfo cellStateTensorInfo({ batchSize, numUnits}, armnn::DataType::Float32);
2414 armnn::TensorInfo outputStateTensorInfo({ batchSize, outputSize }, armnn::DataType::Float32);
2415 armnn::TensorInfo outputTensorInfo({ batchSize, timeSize, outputSize }, armnn::DataType::Float32);
2416
2417 inputLayer->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(0));
2418 inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
2419
2420 outputStateIn->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(1));
2421 outputStateIn->GetOutputSlot(0).SetTensorInfo(outputStateTensorInfo);
2422
2423 cellStateIn->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(2));
2424 cellStateIn->GetOutputSlot(0).SetTensorInfo(cellStateTensorInfo);
2425
2426 unidirectionalSequenceLstmLayer->GetOutputSlot(0).Connect(outputLayer->GetInputSlot(0));
2427 unidirectionalSequenceLstmLayer->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
2428
2429 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
2430 CHECK(deserializedNetwork);
2431
2432 VerifyLstmLayer<armnn::UnidirectionalSequenceLstmDescriptor> checker(
2433 layerName,
2434 {inputTensorInfo, outputStateTensorInfo, cellStateTensorInfo},
2435 {outputTensorInfo},
2436 descriptor,
2437 params);
2438 deserializedNetwork->ExecuteStrategy(checker);
2439}
2440
2441TEST_CASE("SerializeDeserializeUnidirectionalSequenceLstmNoCifgWithPeepholeWithProjectionWithLayerNorm")
2442{
2443 armnn::UnidirectionalSequenceLstmDescriptor descriptor;
2444 descriptor.m_ActivationFunc = 4;
2445 descriptor.m_ClippingThresProj = 0.0f;
2446 descriptor.m_ClippingThresCell = 0.0f;
2447 descriptor.m_CifgEnabled = false; // if this is true then we DON'T need to set the OptCifgParams
2448 descriptor.m_ProjectionEnabled = true;
2449 descriptor.m_PeepholeEnabled = true;
2450 descriptor.m_LayerNormEnabled = true;
2451 descriptor.m_TimeMajor = false;
2452
2453 const uint32_t batchSize = 2;
2454 const uint32_t timeSize = 2;
2455 const uint32_t inputSize = 5;
2456 const uint32_t numUnits = 20;
2457 const uint32_t outputSize = 16;
2458
2459 armnn::TensorInfo tensorInfo20x5({numUnits, inputSize}, armnn::DataType::Float32);
2460 std::vector<float> inputToInputWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
2461 armnn::ConstTensor inputToInputWeights(tensorInfo20x5, inputToInputWeightsData);
2462
2463 std::vector<float> inputToForgetWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
2464 armnn::ConstTensor inputToForgetWeights(tensorInfo20x5, inputToForgetWeightsData);
2465
2466 std::vector<float> inputToCellWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
2467 armnn::ConstTensor inputToCellWeights(tensorInfo20x5, inputToCellWeightsData);
2468
2469 std::vector<float> inputToOutputWeightsData = GenerateRandomData<float>(tensorInfo20x5.GetNumElements());
2470 armnn::ConstTensor inputToOutputWeights(tensorInfo20x5, inputToOutputWeightsData);
2471
2472 armnn::TensorInfo tensorInfo20({numUnits}, armnn::DataType::Float32);
2473 std::vector<float> inputGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2474 armnn::ConstTensor inputGateBias(tensorInfo20, inputGateBiasData);
2475
2476 std::vector<float> forgetGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2477 armnn::ConstTensor forgetGateBias(tensorInfo20, forgetGateBiasData);
2478
2479 std::vector<float> cellBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2480 armnn::ConstTensor cellBias(tensorInfo20, cellBiasData);
2481
2482 std::vector<float> outputGateBiasData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2483 armnn::ConstTensor outputGateBias(tensorInfo20, outputGateBiasData);
2484
2485 armnn::TensorInfo tensorInfo20x16({numUnits, outputSize}, armnn::DataType::Float32);
2486 std::vector<float> recurrentToInputWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
2487 armnn::ConstTensor recurrentToInputWeights(tensorInfo20x16, recurrentToInputWeightsData);
2488
2489 std::vector<float> recurrentToForgetWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
2490 armnn::ConstTensor recurrentToForgetWeights(tensorInfo20x16, recurrentToForgetWeightsData);
2491
2492 std::vector<float> recurrentToCellWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
2493 armnn::ConstTensor recurrentToCellWeights(tensorInfo20x16, recurrentToCellWeightsData);
2494
2495 std::vector<float> recurrentToOutputWeightsData = GenerateRandomData<float>(tensorInfo20x16.GetNumElements());
2496 armnn::ConstTensor recurrentToOutputWeights(tensorInfo20x16, recurrentToOutputWeightsData);
2497
2498 std::vector<float> cellToInputWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2499 armnn::ConstTensor cellToInputWeights(tensorInfo20, cellToInputWeightsData);
2500
2501 std::vector<float> cellToForgetWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2502 armnn::ConstTensor cellToForgetWeights(tensorInfo20, cellToForgetWeightsData);
2503
2504 std::vector<float> cellToOutputWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2505 armnn::ConstTensor cellToOutputWeights(tensorInfo20, cellToOutputWeightsData);
2506
2507 armnn::TensorInfo tensorInfo16x20({outputSize, numUnits}, armnn::DataType::Float32);
2508 std::vector<float> projectionWeightsData = GenerateRandomData<float>(tensorInfo16x20.GetNumElements());
2509 armnn::ConstTensor projectionWeights(tensorInfo16x20, projectionWeightsData);
2510
2511 armnn::TensorInfo tensorInfo16({outputSize}, armnn::DataType::Float32);
2512 std::vector<float> projectionBiasData(outputSize, 0.f);
2513 armnn::ConstTensor projectionBias(tensorInfo16, projectionBiasData);
2514
2515 std::vector<float> inputLayerNormWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2516 armnn::ConstTensor inputLayerNormWeights(tensorInfo20, forgetGateBiasData);
2517
2518 std::vector<float> forgetLayerNormWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2519 armnn::ConstTensor forgetLayerNormWeights(tensorInfo20, forgetGateBiasData);
2520
2521 std::vector<float> cellLayerNormWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2522 armnn::ConstTensor cellLayerNormWeights(tensorInfo20, forgetGateBiasData);
2523
2524 std::vector<float> outLayerNormWeightsData = GenerateRandomData<float>(tensorInfo20.GetNumElements());
2525 armnn::ConstTensor outLayerNormWeights(tensorInfo20, forgetGateBiasData);
2526
2527 armnn::LstmInputParams params;
2528 params.m_InputToForgetWeights = &inputToForgetWeights;
2529 params.m_InputToCellWeights = &inputToCellWeights;
2530 params.m_InputToOutputWeights = &inputToOutputWeights;
2531 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
2532 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
2533 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
2534 params.m_ForgetGateBias = &forgetGateBias;
2535 params.m_CellBias = &cellBias;
2536 params.m_OutputGateBias = &outputGateBias;
2537
2538 // additional params because: descriptor.m_CifgEnabled = false
2539 params.m_InputToInputWeights = &inputToInputWeights;
2540 params.m_RecurrentToInputWeights = &recurrentToInputWeights;
2541 params.m_CellToInputWeights = &cellToInputWeights;
2542 params.m_InputGateBias = &inputGateBias;
2543
2544 // additional params because: descriptor.m_ProjectionEnabled = true
2545 params.m_ProjectionWeights = &projectionWeights;
2546 params.m_ProjectionBias = &projectionBias;
2547
2548 // additional params because: descriptor.m_PeepholeEnabled = true
2549 params.m_CellToForgetWeights = &cellToForgetWeights;
2550 params.m_CellToOutputWeights = &cellToOutputWeights;
2551
2552 // additional params because: despriptor.m_LayerNormEnabled = true
2553 params.m_InputLayerNormWeights = &inputLayerNormWeights;
2554 params.m_ForgetLayerNormWeights = &forgetLayerNormWeights;
2555 params.m_CellLayerNormWeights = &cellLayerNormWeights;
2556 params.m_OutputLayerNormWeights = &outLayerNormWeights;
2557
2558 armnn::INetworkPtr network = armnn::INetwork::Create();
2559 armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(0);
2560 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(1);
2561 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(2);
2562 const std::string layerName("unidirectionalSequenceLstm");
2563 armnn::IConnectableLayer* const unidirectionalSequenceLstmLayer =
2564 network->AddUnidirectionalSequenceLstmLayer(descriptor, params, layerName.c_str());
2565 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(0);
2566
2567 // connect up
2568 armnn::TensorInfo inputTensorInfo({ batchSize, timeSize, inputSize }, armnn::DataType::Float32);
2569 armnn::TensorInfo cellStateTensorInfo({ batchSize, numUnits}, armnn::DataType::Float32);
2570 armnn::TensorInfo outputStateTensorInfo({ batchSize, outputSize }, armnn::DataType::Float32);
2571 armnn::TensorInfo outputTensorInfo({ batchSize, timeSize, outputSize }, armnn::DataType::Float32);
2572
2573 inputLayer->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(0));
2574 inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
2575
2576 outputStateIn->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(1));
2577 outputStateIn->GetOutputSlot(0).SetTensorInfo(outputStateTensorInfo);
2578
2579 cellStateIn->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(2));
2580 cellStateIn->GetOutputSlot(0).SetTensorInfo(cellStateTensorInfo);
2581
2582 unidirectionalSequenceLstmLayer->GetOutputSlot(0).Connect(outputLayer->GetInputSlot(0));
2583 unidirectionalSequenceLstmLayer->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
2584
2585 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
2586 CHECK(deserializedNetwork);
2587
2588 VerifyLstmLayer<armnn::UnidirectionalSequenceLstmDescriptor> checker(
2589 layerName,
2590 {inputTensorInfo, outputStateTensorInfo, cellStateTensorInfo},
2591 {outputTensorInfo},
2592 descriptor,
2593 params);
2594 deserializedNetwork->ExecuteStrategy(checker);
2595}
2596
2597TEST_CASE("SerializeDeserializeUnidirectionalSequenceLstmCifgPeepholeNoProjectionTimeMajor")
2598{
2599 armnn::UnidirectionalSequenceLstmDescriptor descriptor;
2600 descriptor.m_ActivationFunc = 4;
2601 descriptor.m_ClippingThresProj = 0.0f;
2602 descriptor.m_ClippingThresCell = 0.0f;
2603 descriptor.m_CifgEnabled = true; // if this is true then we DON'T need to set the OptCifgParams
2604 descriptor.m_ProjectionEnabled = false;
2605 descriptor.m_PeepholeEnabled = true;
2606 descriptor.m_TimeMajor = true;
2607
2608 const uint32_t batchSize = 1;
2609 const uint32_t timeSize = 2;
2610 const uint32_t inputSize = 2;
2611 const uint32_t numUnits = 4;
2612 const uint32_t outputSize = numUnits;
2613
2614 armnn::TensorInfo inputWeightsInfo1({numUnits, inputSize}, armnn::DataType::Float32);
2615 std::vector<float> inputToForgetWeightsData = GenerateRandomData<float>(inputWeightsInfo1.GetNumElements());
2616 armnn::ConstTensor inputToForgetWeights(inputWeightsInfo1, inputToForgetWeightsData);
2617
2618 std::vector<float> inputToCellWeightsData = GenerateRandomData<float>(inputWeightsInfo1.GetNumElements());
2619 armnn::ConstTensor inputToCellWeights(inputWeightsInfo1, inputToCellWeightsData);
2620
2621 std::vector<float> inputToOutputWeightsData = GenerateRandomData<float>(inputWeightsInfo1.GetNumElements());
2622 armnn::ConstTensor inputToOutputWeights(inputWeightsInfo1, inputToOutputWeightsData);
2623
2624 armnn::TensorInfo inputWeightsInfo2({numUnits, outputSize}, armnn::DataType::Float32);
2625 std::vector<float> recurrentToForgetWeightsData = GenerateRandomData<float>(inputWeightsInfo2.GetNumElements());
2626 armnn::ConstTensor recurrentToForgetWeights(inputWeightsInfo2, recurrentToForgetWeightsData);
2627
2628 std::vector<float> recurrentToCellWeightsData = GenerateRandomData<float>(inputWeightsInfo2.GetNumElements());
2629 armnn::ConstTensor recurrentToCellWeights(inputWeightsInfo2, recurrentToCellWeightsData);
2630
2631 std::vector<float> recurrentToOutputWeightsData = GenerateRandomData<float>(inputWeightsInfo2.GetNumElements());
2632 armnn::ConstTensor recurrentToOutputWeights(inputWeightsInfo2, recurrentToOutputWeightsData);
2633
2634 armnn::TensorInfo inputWeightsInfo3({numUnits}, armnn::DataType::Float32);
2635 std::vector<float> cellToForgetWeightsData = GenerateRandomData<float>(inputWeightsInfo3.GetNumElements());
2636 armnn::ConstTensor cellToForgetWeights(inputWeightsInfo3, cellToForgetWeightsData);
2637
2638 std::vector<float> cellToOutputWeightsData = GenerateRandomData<float>(inputWeightsInfo3.GetNumElements());
2639 armnn::ConstTensor cellToOutputWeights(inputWeightsInfo3, cellToOutputWeightsData);
2640
2641 std::vector<float> forgetGateBiasData(numUnits, 1.0f);
2642 armnn::ConstTensor forgetGateBias(inputWeightsInfo3, forgetGateBiasData);
2643
2644 std::vector<float> cellBiasData(numUnits, 0.0f);
2645 armnn::ConstTensor cellBias(inputWeightsInfo3, cellBiasData);
2646
2647 std::vector<float> outputGateBiasData(numUnits, 0.0f);
2648 armnn::ConstTensor outputGateBias(inputWeightsInfo3, outputGateBiasData);
2649
2650 armnn::LstmInputParams params;
2651 params.m_InputToForgetWeights = &inputToForgetWeights;
2652 params.m_InputToCellWeights = &inputToCellWeights;
2653 params.m_InputToOutputWeights = &inputToOutputWeights;
2654 params.m_RecurrentToForgetWeights = &recurrentToForgetWeights;
2655 params.m_RecurrentToCellWeights = &recurrentToCellWeights;
2656 params.m_RecurrentToOutputWeights = &recurrentToOutputWeights;
2657 params.m_ForgetGateBias = &forgetGateBias;
2658 params.m_CellBias = &cellBias;
2659 params.m_OutputGateBias = &outputGateBias;
2660 params.m_CellToForgetWeights = &cellToForgetWeights;
2661 params.m_CellToOutputWeights = &cellToOutputWeights;
2662
2663 armnn::INetworkPtr network = armnn::INetwork::Create();
2664 armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(0);
2665 armnn::IConnectableLayer* const cellStateIn = network->AddInputLayer(1);
2666 armnn::IConnectableLayer* const outputStateIn = network->AddInputLayer(2);
2667 const std::string layerName("UnidirectionalSequenceLstm");
2668 armnn::IConnectableLayer* const unidirectionalSequenceLstmLayer =
2669 network->AddUnidirectionalSequenceLstmLayer(descriptor, params, layerName.c_str());
2670 armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(0);
2671
2672 // connect up
2673 armnn::TensorInfo inputTensorInfo({ timeSize, batchSize, inputSize }, armnn::DataType::Float32);
2674 armnn::TensorInfo cellStateTensorInfo({ batchSize, numUnits}, armnn::DataType::Float32);
2675 armnn::TensorInfo outputStateTensorInfo({ batchSize, outputSize }, armnn::DataType::Float32);
2676 armnn::TensorInfo outputTensorInfo({ timeSize, batchSize, outputSize }, armnn::DataType::Float32);
2677
2678 inputLayer->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(0));
2679 inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
2680
2681 outputStateIn->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(1));
2682 outputStateIn->GetOutputSlot(0).SetTensorInfo(outputStateTensorInfo);
2683
2684 cellStateIn->GetOutputSlot(0).Connect(unidirectionalSequenceLstmLayer->GetInputSlot(2));
2685 cellStateIn->GetOutputSlot(0).SetTensorInfo(cellStateTensorInfo);
2686
2687 unidirectionalSequenceLstmLayer->GetOutputSlot(0).Connect(outputLayer->GetInputSlot(0));
2688 unidirectionalSequenceLstmLayer->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
2689
2690 armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
2691 CHECK(deserializedNetwork);
2692
2693 VerifyLstmLayer<armnn::UnidirectionalSequenceLstmDescriptor> checker(
2694 layerName,
2695 {inputTensorInfo, outputStateTensorInfo, cellStateTensorInfo},
2696 {outputTensorInfo},
2697 descriptor,
2698 params);
2699 deserializedNetwork->ExecuteStrategy(checker);
2700}
2701
Sadik Armagan1625efc2021-06-10 18:24:34 +0100[diff] [blame]2702}