blob: 1fba3d4b46852b27d893d3d7276f8f1f8f50073d [file] [log] [blame]
Anthony Barbier6ff3b192017-09-04 18:44:23 +01001/*
Ioan-Cristian Szabo91d20d92017-10-27 17:35:40 +01002 * Copyright (c) 2017-2018 ARM Limited.
Anthony Barbier6ff3b192017-09-04 18:44:23 +01003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#ifndef __ARM_COMPUTE_TEST_TENSOR_LIBRARY_H__
25#define __ARM_COMPUTE_TEST_TENSOR_LIBRARY_H__
26
Anthony Barbier6ff3b192017-09-04 18:44:23 +010027#include "arm_compute/core/Coordinates.h"
28#include "arm_compute/core/Error.h"
29#include "arm_compute/core/Helpers.h"
30#include "arm_compute/core/TensorInfo.h"
31#include "arm_compute/core/TensorShape.h"
32#include "arm_compute/core/Types.h"
33#include "arm_compute/core/Window.h"
SiCong Li86b53332017-08-23 11:02:43 +010034#include "libnpy/npy.hpp"
Moritz Pflanzere49e2662017-07-21 15:55:28 +010035#include "tests/RawTensor.h"
36#include "tests/TensorCache.h"
37#include "tests/Utils.h"
Anthony Barbierf6705ec2017-09-28 12:01:10 +010038#include "tests/framework/Exceptions.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010039
40#include <algorithm>
41#include <cstddef>
42#include <fstream>
43#include <random>
44#include <string>
45#include <type_traits>
SiCong Li86b53332017-08-23 11:02:43 +010046#include <vector>
Anthony Barbier6ff3b192017-09-04 18:44:23 +010047
48namespace arm_compute
49{
50namespace test
51{
52/** Factory class to create and fill tensors.
53 *
54 * Allows to initialise tensors from loaded images or by specifying the shape
55 * explicitly. Furthermore, provides methods to fill tensors with the content of
56 * loaded images or with random values.
57 */
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +010058class AssetsLibrary final
Anthony Barbier6ff3b192017-09-04 18:44:23 +010059{
60public:
John Richardson70f946b2017-10-02 16:52:16 +010061 /** Initialises the library with a @p path to the assets directory.
Anthony Barbier6ff3b192017-09-04 18:44:23 +010062 * Furthermore, sets the seed for the random generator to @p seed.
63 *
John Richardson70f946b2017-10-02 16:52:16 +010064 * @param[in] path Path to load assets from.
Anthony Barbier6ff3b192017-09-04 18:44:23 +010065 * @param[in] seed Seed used to initialise the random number generator.
66 */
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +010067 AssetsLibrary(std::string path, std::random_device::result_type seed);
Anthony Barbier6ff3b192017-09-04 18:44:23 +010068
Alex Gildayc357c472018-03-21 13:54:09 +000069 /** Path to assets directory used to initialise library.
70 *
71 * @return the path to the assets directory.
72 */
John Richardson70f946b2017-10-02 16:52:16 +010073 std::string path() const;
74
Alex Gildayc357c472018-03-21 13:54:09 +000075 /** Seed that is used to fill tensors with random values.
76 *
77 * @return the initial random seed.
78 */
Anthony Barbier6ff3b192017-09-04 18:44:23 +010079 std::random_device::result_type seed() const;
80
Giorgio Arenafda46182017-06-16 13:57:33 +010081 /** Provides a tensor shape for the specified image.
82 *
83 * @param[in] name Image file used to look up the raw tensor.
Alex Gildayc357c472018-03-21 13:54:09 +000084 *
85 * @return the tensor shape for the specified image.
Giorgio Arenafda46182017-06-16 13:57:33 +010086 */
87 TensorShape get_image_shape(const std::string &name);
88
Alex Gildayc357c472018-03-21 13:54:09 +000089 /** Provides a constant raw tensor for the specified image.
Anthony Barbier6ff3b192017-09-04 18:44:23 +010090 *
91 * @param[in] name Image file used to look up the raw tensor.
Alex Gildayc357c472018-03-21 13:54:09 +000092 *
93 * @return a raw tensor for the specified image.
Anthony Barbier6ff3b192017-09-04 18:44:23 +010094 */
95 const RawTensor &get(const std::string &name) const;
96
97 /** Provides a raw tensor for the specified image.
98 *
99 * @param[in] name Image file used to look up the raw tensor.
Alex Gildayc357c472018-03-21 13:54:09 +0000100 *
101 * @return a raw tensor for the specified image.
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100102 */
103 RawTensor get(const std::string &name);
104
105 /** Creates an uninitialised raw tensor with the given @p data_type and @p
106 * num_channels. The shape is derived from the specified image.
107 *
108 * @param[in] name Image file used to initialise the tensor.
109 * @param[in] data_type Data type used to initialise the tensor.
110 * @param[in] num_channels Number of channels used to initialise the tensor.
Alex Gildayc357c472018-03-21 13:54:09 +0000111 *
112 * @return a raw tensor for the specified image.
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100113 */
114 RawTensor get(const std::string &name, DataType data_type, int num_channels = 1) const;
115
116 /** Provides a contant raw tensor for the specified image after it has been
117 * converted to @p format.
118 *
119 * @param[in] name Image file used to look up the raw tensor.
120 * @param[in] format Format used to look up the raw tensor.
Alex Gildayc357c472018-03-21 13:54:09 +0000121 *
122 * @return a raw tensor for the specified image.
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100123 */
124 const RawTensor &get(const std::string &name, Format format) const;
125
126 /** Provides a raw tensor for the specified image after it has been
127 * converted to @p format.
128 *
129 * @param[in] name Image file used to look up the raw tensor.
130 * @param[in] format Format used to look up the raw tensor.
Alex Gildayc357c472018-03-21 13:54:09 +0000131 *
132 * @return a raw tensor for the specified image.
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100133 */
134 RawTensor get(const std::string &name, Format format);
135
136 /** Provides a contant raw tensor for the specified channel after it has
137 * been extracted form the given image.
138 *
139 * @param[in] name Image file used to look up the raw tensor.
140 * @param[in] channel Channel used to look up the raw tensor.
141 *
142 * @note The channel has to be unambiguous so that the format can be
143 * inferred automatically.
Alex Gildayc357c472018-03-21 13:54:09 +0000144 *
145 * @return a raw tensor for the specified image channel.
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100146 */
147 const RawTensor &get(const std::string &name, Channel channel) const;
148
149 /** Provides a raw tensor for the specified channel after it has been
150 * extracted form the given image.
151 *
152 * @param[in] name Image file used to look up the raw tensor.
153 * @param[in] channel Channel used to look up the raw tensor.
154 *
155 * @note The channel has to be unambiguous so that the format can be
156 * inferred automatically.
Alex Gildayc357c472018-03-21 13:54:09 +0000157 *
158 * @return a raw tensor for the specified image channel.
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100159 */
160 RawTensor get(const std::string &name, Channel channel);
161
162 /** Provides a constant raw tensor for the specified channel after it has
163 * been extracted form the given image formatted to @p format.
164 *
165 * @param[in] name Image file used to look up the raw tensor.
166 * @param[in] format Format used to look up the raw tensor.
167 * @param[in] channel Channel used to look up the raw tensor.
Alex Gildayc357c472018-03-21 13:54:09 +0000168 *
169 * @return a raw tensor for the specified image channel.
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100170 */
171 const RawTensor &get(const std::string &name, Format format, Channel channel) const;
172
173 /** Provides a raw tensor for the specified channel after it has been
174 * extracted form the given image formatted to @p format.
175 *
176 * @param[in] name Image file used to look up the raw tensor.
177 * @param[in] format Format used to look up the raw tensor.
178 * @param[in] channel Channel used to look up the raw tensor.
Alex Gildayc357c472018-03-21 13:54:09 +0000179 *
180 * @return a raw tensor for the specified image channel.
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100181 */
182 RawTensor get(const std::string &name, Format format, Channel channel);
183
Giorgio Arenaa2611812017-07-21 10:08:48 +0100184 /** Puts garbage values all around the tensor for testing purposes
185 *
186 * @param[in, out] tensor To be filled tensor.
187 * @param[in] distribution Distribution used to fill the tensor's surroundings.
188 * @param[in] seed_offset The offset will be added to the global seed before initialising the random generator.
189 */
190 template <typename T, typename D>
191 void fill_borders_with_garbage(T &&tensor, D &&distribution, std::random_device::result_type seed_offset) const;
192
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100193 /** Fills the specified @p tensor with random values drawn from @p
194 * distribution.
195 *
196 * @param[in, out] tensor To be filled tensor.
197 * @param[in] distribution Distribution used to fill the tensor.
198 * @param[in] seed_offset The offset will be added to the global seed before initialising the random generator.
199 *
200 * @note The @p distribution has to provide operator(Generator &) which
201 * will be used to draw samples.
202 */
203 template <typename T, typename D>
204 void fill(T &&tensor, D &&distribution, std::random_device::result_type seed_offset) const;
205
206 /** Fills the specified @p raw tensor with random values drawn from @p
207 * distribution.
208 *
209 * @param[in, out] raw To be filled raw.
210 * @param[in] distribution Distribution used to fill the tensor.
211 * @param[in] seed_offset The offset will be added to the global seed before initialising the random generator.
212 *
213 * @note The @p distribution has to provide operator(Generator &) which
214 * will be used to draw samples.
215 */
216 template <typename D>
217 void fill(RawTensor &raw, D &&distribution, std::random_device::result_type seed_offset) const;
218
219 /** Fills the specified @p tensor with the content of the specified image
220 * converted to the given format.
221 *
222 * @param[in, out] tensor To be filled tensor.
223 * @param[in] name Image file used to fill the tensor.
224 * @param[in] format Format of the image used to fill the tensor.
225 *
226 * @warning No check is performed that the specified format actually
227 * matches the format of the tensor.
228 */
229 template <typename T>
230 void fill(T &&tensor, const std::string &name, Format format) const;
231
232 /** Fills the raw tensor with the content of the specified image
233 * converted to the given format.
234 *
235 * @param[in, out] raw To be filled raw tensor.
236 * @param[in] name Image file used to fill the tensor.
237 * @param[in] format Format of the image used to fill the tensor.
238 *
239 * @warning No check is performed that the specified format actually
240 * matches the format of the tensor.
241 */
242 void fill(RawTensor &raw, const std::string &name, Format format) const;
243
244 /** Fills the specified @p tensor with the content of the specified channel
245 * extracted from the given image.
246 *
247 * @param[in, out] tensor To be filled tensor.
248 * @param[in] name Image file used to fill the tensor.
249 * @param[in] channel Channel of the image used to fill the tensor.
250 *
251 * @note The channel has to be unambiguous so that the format can be
252 * inferred automatically.
253 *
254 * @warning No check is performed that the specified format actually
255 * matches the format of the tensor.
256 */
257 template <typename T>
258 void fill(T &&tensor, const std::string &name, Channel channel) const;
259
260 /** Fills the raw tensor with the content of the specified channel
261 * extracted from the given image.
262 *
263 * @param[in, out] raw To be filled raw tensor.
264 * @param[in] name Image file used to fill the tensor.
265 * @param[in] channel Channel of the image used to fill the tensor.
266 *
267 * @note The channel has to be unambiguous so that the format can be
268 * inferred automatically.
269 *
270 * @warning No check is performed that the specified format actually
271 * matches the format of the tensor.
272 */
273 void fill(RawTensor &raw, const std::string &name, Channel channel) const;
274
275 /** Fills the specified @p tensor with the content of the specified channel
276 * extracted from the given image after it has been converted to the given
277 * format.
278 *
279 * @param[in, out] tensor To be filled tensor.
280 * @param[in] name Image file used to fill the tensor.
281 * @param[in] format Format of the image used to fill the tensor.
282 * @param[in] channel Channel of the image used to fill the tensor.
283 *
284 * @warning No check is performed that the specified format actually
285 * matches the format of the tensor.
286 */
287 template <typename T>
288 void fill(T &&tensor, const std::string &name, Format format, Channel channel) const;
289
290 /** Fills the raw tensor with the content of the specified channel
291 * extracted from the given image after it has been converted to the given
292 * format.
293 *
294 * @param[in, out] raw To be filled raw tensor.
295 * @param[in] name Image file used to fill the tensor.
296 * @param[in] format Format of the image used to fill the tensor.
297 * @param[in] channel Channel of the image used to fill the tensor.
298 *
299 * @warning No check is performed that the specified format actually
300 * matches the format of the tensor.
301 */
302 void fill(RawTensor &raw, const std::string &name, Format format, Channel channel) const;
303
Alex Gilday345ab182018-01-09 11:40:19 +0000304 /** Fills the specified @p tensor with the content of the raw tensor.
305 *
306 * @param[in, out] tensor To be filled tensor.
307 * @param[in] raw Raw tensor used to fill the tensor.
308 *
309 * @warning No check is performed that the specified format actually
310 * matches the format of the tensor.
311 */
312 template <typename T>
313 void fill(T &&tensor, RawTensor raw) const;
314
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100315 /** Fill a tensor with uniform distribution across the range of its type
316 *
317 * @param[in, out] tensor To be filled tensor.
318 * @param[in] seed_offset The offset will be added to the global seed before initialising the random generator.
319 */
320 template <typename T>
321 void fill_tensor_uniform(T &&tensor, std::random_device::result_type seed_offset) const;
322
323 /** Fill a tensor with uniform distribution across the a specified range
324 *
325 * @param[in, out] tensor To be filled tensor.
326 * @param[in] seed_offset The offset will be added to the global seed before initialising the random generator.
327 * @param[in] low lowest value in the range (inclusive)
328 * @param[in] high highest value in the range (inclusive)
329 *
330 * @note @p low and @p high must be of the same type as the data type of @p tensor
331 */
332 template <typename T, typename D>
333 void fill_tensor_uniform(T &&tensor, std::random_device::result_type seed_offset, D low, D high) const;
334
SiCong Li86b53332017-08-23 11:02:43 +0100335 /** Fills the specified @p tensor with data loaded from .npy (numpy binary) in specified path.
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100336 *
337 * @param[in, out] tensor To be filled tensor.
338 * @param[in] name Data file.
SiCong Li86b53332017-08-23 11:02:43 +0100339 *
340 * @note The numpy array stored in the binary .npy file must be row-major in the sense that it
341 * must store elements within a row consecutively in the memory, then rows within a 2D slice,
342 * then 2D slices within a 3D slice and so on. Note that it imposes no restrictions on what
343 * indexing convention is used in the numpy array. That is, the numpy array can be either fortran
344 * style or C style as long as it adheres to the rule above.
345 *
346 * More concretely, the orders of dimensions for each style are as follows:
347 * C-style (numpy default):
348 * array[HigherDims..., Z, Y, X]
349 * Fortran style:
350 * array[X, Y, Z, HigherDims...]
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100351 */
352 template <typename T>
353 void fill_layer_data(T &&tensor, std::string name) const;
354
Michele Di Giorgio4d336302018-03-02 09:43:54 +0000355 /** Fill a tensor with a constant value
356 *
357 * @param[in, out] tensor To be filled tensor.
358 * @param[in] value Value to be assigned to all elements of the input tensor.
359 *
360 * @note @p value must be of the same type as the data type of @p tensor
361 */
362 template <typename T, typename D>
363 void fill_tensor_value(T &&tensor, D value) const;
364
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100365private:
366 // Function prototype to convert between image formats.
367 using Converter = void (*)(const RawTensor &src, RawTensor &dst);
368 // Function prototype to extract a channel from an image.
369 using Extractor = void (*)(const RawTensor &src, RawTensor &dst);
370 // Function prototype to load an image file.
371 using Loader = RawTensor (*)(const std::string &path);
372
373 const Converter &get_converter(Format src, Format dst) const;
374 const Converter &get_converter(DataType src, Format dst) const;
375 const Converter &get_converter(Format src, DataType dst) const;
376 const Converter &get_converter(DataType src, DataType dst) const;
377 const Extractor &get_extractor(Format format, Channel) const;
378 const Loader &get_loader(const std::string &extension) const;
379
380 /** Creates a raw tensor from the specified image.
381 *
382 * @param[in] name To be loaded image file.
383 *
384 * @note If use_single_image is true @p name is ignored and the user image
385 * is loaded instead.
386 */
387 RawTensor load_image(const std::string &name) const;
388
389 /** Provides a raw tensor for the specified image and format.
390 *
391 * @param[in] name Image file used to look up the raw tensor.
392 * @param[in] format Format used to look up the raw tensor.
393 *
394 * If the tensor has already been requested before the cached version will
395 * be returned. Otherwise the tensor will be added to the cache.
396 *
397 * @note If use_single_image is true @p name is ignored and the user image
398 * is loaded instead.
399 */
400 const RawTensor &find_or_create_raw_tensor(const std::string &name, Format format) const;
401
402 /** Provides a raw tensor for the specified image, format and channel.
403 *
404 * @param[in] name Image file used to look up the raw tensor.
405 * @param[in] format Format used to look up the raw tensor.
406 * @param[in] channel Channel used to look up the raw tensor.
407 *
408 * If the tensor has already been requested before the cached version will
409 * be returned. Otherwise the tensor will be added to the cache.
410 *
411 * @note If use_single_image is true @p name is ignored and the user image
412 * is loaded instead.
413 */
414 const RawTensor &find_or_create_raw_tensor(const std::string &name, Format format, Channel channel) const;
415
416 mutable TensorCache _cache{};
417 mutable std::mutex _format_lock{};
418 mutable std::mutex _channel_lock{};
Anthony Barbierac69aa12017-07-03 17:39:37 +0100419 const std::string _library_path;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100420 std::random_device::result_type _seed;
421};
422
423template <typename T, typename D>
Giorgio Arenaa2611812017-07-21 10:08:48 +0100424void AssetsLibrary::fill_borders_with_garbage(T &&tensor, D &&distribution, std::random_device::result_type seed_offset) const
425{
426 const PaddingSize padding_size = tensor.padding();
427
428 Window window;
429 window.set(0, Window::Dimension(-padding_size.left, tensor.shape()[0] + padding_size.right, 1));
Gian Marco5420b282017-11-29 10:41:38 +0000430 if(tensor.shape().num_dimensions() > 1)
431 {
432 window.set(1, Window::Dimension(-padding_size.top, tensor.shape()[1] + padding_size.bottom, 1));
433 }
Giorgio Arenaa2611812017-07-21 10:08:48 +0100434
435 std::mt19937 gen(_seed);
436
437 execute_window_loop(window, [&](const Coordinates & id)
438 {
439 TensorShape shape = tensor.shape();
440
441 // If outside of valid region
442 if(id.x() < 0 || id.x() >= static_cast<int>(shape.x()) || id.y() < 0 || id.y() >= static_cast<int>(shape.y()))
443 {
444 using ResultType = typename std::remove_reference<D>::type::result_type;
445 const ResultType value = distribution(gen);
446 void *const out_ptr = tensor(id);
447 store_value_with_data_type(out_ptr, value, tensor.data_type());
448 }
449 });
450}
451
452template <typename T, typename D>
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +0100453void AssetsLibrary::fill(T &&tensor, D &&distribution, std::random_device::result_type seed_offset) const
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100454{
Ioan-Cristian Szabo9414f642017-10-27 17:35:40 +0100455 using ResultType = typename std::remove_reference<D>::type::result_type;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100456
457 std::mt19937 gen(_seed + seed_offset);
458
Giorgio Arena563494c2018-04-30 17:29:41 +0100459 const bool is_nhwc = tensor.data_layout() == DataLayout::NHWC;
460 TensorShape shape(tensor.shape());
461
462 if(is_nhwc)
463 {
464 // Ensure that the equivalent tensors will be filled for both data layouts
465 permute(shape, PermutationVector(1U, 2U, 0U));
466 }
467
Ioan-Cristian Szabo9414f642017-10-27 17:35:40 +0100468 // Iterate over all elements
469 for(int element_idx = 0; element_idx < tensor.num_elements(); ++element_idx)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100470 {
Giorgio Arena563494c2018-04-30 17:29:41 +0100471 Coordinates id = index2coord(shape, element_idx);
472
473 if(is_nhwc)
474 {
475 // Write in the correct id for permuted shapes
476 permute(id, PermutationVector(2U, 0U, 1U));
477 }
Ioan-Cristian Szabo9414f642017-10-27 17:35:40 +0100478
479 // Iterate over all channels
480 for(int channel = 0; channel < tensor.num_channels(); ++channel)
481 {
482 const ResultType value = distribution(gen);
483 ResultType &target_value = reinterpret_cast<ResultType *const>(tensor(id))[channel];
484
485 store_value_with_data_type(&target_value, value, tensor.data_type());
486 }
487 }
Giorgio Arenaa2611812017-07-21 10:08:48 +0100488
489 fill_borders_with_garbage(tensor, distribution, seed_offset);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100490}
491
492template <typename D>
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +0100493void AssetsLibrary::fill(RawTensor &raw, D &&distribution, std::random_device::result_type seed_offset) const
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100494{
495 std::mt19937 gen(_seed + seed_offset);
496
497 for(size_t offset = 0; offset < raw.size(); offset += raw.element_size())
498 {
499 using ResultType = typename std::remove_reference<D>::type::result_type;
500 const ResultType value = distribution(gen);
501 store_value_with_data_type(raw.data() + offset, value, raw.data_type());
502 }
503}
504
505template <typename T>
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +0100506void AssetsLibrary::fill(T &&tensor, const std::string &name, Format format) const
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100507{
508 const RawTensor &raw = get(name, format);
509
510 for(size_t offset = 0; offset < raw.size(); offset += raw.element_size())
511 {
512 const Coordinates id = index2coord(raw.shape(), offset / raw.element_size());
513
Moritz Pflanzer82e70a12017-08-08 16:20:45 +0100514 const RawTensor::value_type *const raw_ptr = raw.data() + offset;
515 const auto out_ptr = static_cast<RawTensor::value_type *>(tensor(id));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100516 std::copy_n(raw_ptr, raw.element_size(), out_ptr);
517 }
518}
519
520template <typename T>
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +0100521void AssetsLibrary::fill(T &&tensor, const std::string &name, Channel channel) const
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100522{
523 fill(std::forward<T>(tensor), name, get_format_for_channel(channel), channel);
524}
525
526template <typename T>
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +0100527void AssetsLibrary::fill(T &&tensor, const std::string &name, Format format, Channel channel) const
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100528{
529 const RawTensor &raw = get(name, format, channel);
530
531 for(size_t offset = 0; offset < raw.size(); offset += raw.element_size())
532 {
533 const Coordinates id = index2coord(raw.shape(), offset / raw.element_size());
534
Moritz Pflanzer82e70a12017-08-08 16:20:45 +0100535 const RawTensor::value_type *const raw_ptr = raw.data() + offset;
536 const auto out_ptr = static_cast<RawTensor::value_type *>(tensor(id));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100537 std::copy_n(raw_ptr, raw.element_size(), out_ptr);
538 }
539}
540
541template <typename T>
Alex Gilday345ab182018-01-09 11:40:19 +0000542void AssetsLibrary::fill(T &&tensor, RawTensor raw) const
543{
544 for(size_t offset = 0; offset < raw.size(); offset += raw.element_size())
545 {
546 const Coordinates id = index2coord(raw.shape(), offset / raw.element_size());
547
548 const RawTensor::value_type *const raw_ptr = raw.data() + offset;
549 const auto out_ptr = static_cast<RawTensor::value_type *>(tensor(id));
550 std::copy_n(raw_ptr, raw.element_size(), out_ptr);
551 }
552}
553
554template <typename T>
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +0100555void AssetsLibrary::fill_tensor_uniform(T &&tensor, std::random_device::result_type seed_offset) const
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100556{
557 switch(tensor.data_type())
558 {
559 case DataType::U8:
Anton Lokhmotovaf6204c2017-11-08 09:34:19 +0000560 case DataType::QASYMM8:
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100561 {
562 std::uniform_int_distribution<uint8_t> distribution_u8(std::numeric_limits<uint8_t>::lowest(), std::numeric_limits<uint8_t>::max());
563 fill(tensor, distribution_u8, seed_offset);
564 break;
565 }
566 case DataType::S8:
567 case DataType::QS8:
568 {
569 std::uniform_int_distribution<int8_t> distribution_s8(std::numeric_limits<int8_t>::lowest(), std::numeric_limits<int8_t>::max());
570 fill(tensor, distribution_s8, seed_offset);
571 break;
572 }
573 case DataType::U16:
574 {
575 std::uniform_int_distribution<uint16_t> distribution_u16(std::numeric_limits<uint16_t>::lowest(), std::numeric_limits<uint16_t>::max());
576 fill(tensor, distribution_u16, seed_offset);
577 break;
578 }
579 case DataType::S16:
Gian Marco Iodicebdb6b0b2017-06-30 12:21:00 +0100580 case DataType::QS16:
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100581 {
582 std::uniform_int_distribution<int16_t> distribution_s16(std::numeric_limits<int16_t>::lowest(), std::numeric_limits<int16_t>::max());
583 fill(tensor, distribution_s16, seed_offset);
584 break;
585 }
586 case DataType::U32:
587 {
588 std::uniform_int_distribution<uint32_t> distribution_u32(std::numeric_limits<uint32_t>::lowest(), std::numeric_limits<uint32_t>::max());
589 fill(tensor, distribution_u32, seed_offset);
590 break;
591 }
592 case DataType::S32:
593 {
594 std::uniform_int_distribution<int32_t> distribution_s32(std::numeric_limits<int32_t>::lowest(), std::numeric_limits<int32_t>::max());
595 fill(tensor, distribution_s32, seed_offset);
596 break;
597 }
598 case DataType::U64:
599 {
600 std::uniform_int_distribution<uint64_t> distribution_u64(std::numeric_limits<uint64_t>::lowest(), std::numeric_limits<uint64_t>::max());
601 fill(tensor, distribution_u64, seed_offset);
602 break;
603 }
604 case DataType::S64:
605 {
606 std::uniform_int_distribution<int64_t> distribution_s64(std::numeric_limits<int64_t>::lowest(), std::numeric_limits<int64_t>::max());
607 fill(tensor, distribution_s64, seed_offset);
608 break;
609 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100610 case DataType::F16:
SiCong Li02dfb2c2017-07-27 17:59:20 +0100611 {
612 // It doesn't make sense to check [-inf, inf], so hard code it to a big number
613 std::uniform_real_distribution<float> distribution_f16(-100.f, 100.f);
614 fill(tensor, distribution_f16, seed_offset);
615 break;
616 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100617 case DataType::F32:
618 {
619 // It doesn't make sense to check [-inf, inf], so hard code it to a big number
620 std::uniform_real_distribution<float> distribution_f32(-1000.f, 1000.f);
621 fill(tensor, distribution_f32, seed_offset);
622 break;
623 }
624 case DataType::F64:
625 {
626 // It doesn't make sense to check [-inf, inf], so hard code it to a big number
627 std::uniform_real_distribution<double> distribution_f64(-1000.f, 1000.f);
628 fill(tensor, distribution_f64, seed_offset);
629 break;
630 }
631 case DataType::SIZET:
632 {
633 std::uniform_int_distribution<size_t> distribution_sizet(std::numeric_limits<size_t>::lowest(), std::numeric_limits<size_t>::max());
634 fill(tensor, distribution_sizet, seed_offset);
635 break;
636 }
637 default:
638 ARM_COMPUTE_ERROR("NOT SUPPORTED!");
639 }
640}
641
642template <typename T, typename D>
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +0100643void AssetsLibrary::fill_tensor_uniform(T &&tensor, std::random_device::result_type seed_offset, D low, D high) const
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100644{
645 switch(tensor.data_type())
646 {
647 case DataType::U8:
Anton Lokhmotovaf6204c2017-11-08 09:34:19 +0000648 case DataType::QASYMM8:
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100649 {
650 ARM_COMPUTE_ERROR_ON(!(std::is_same<uint8_t, D>::value));
651 std::uniform_int_distribution<uint8_t> distribution_u8(low, high);
652 fill(tensor, distribution_u8, seed_offset);
653 break;
654 }
655 case DataType::S8:
656 case DataType::QS8:
657 {
658 ARM_COMPUTE_ERROR_ON(!(std::is_same<int8_t, D>::value));
659 std::uniform_int_distribution<int8_t> distribution_s8(low, high);
660 fill(tensor, distribution_s8, seed_offset);
661 break;
662 }
663 case DataType::U16:
664 {
665 ARM_COMPUTE_ERROR_ON(!(std::is_same<uint16_t, D>::value));
666 std::uniform_int_distribution<uint16_t> distribution_u16(low, high);
667 fill(tensor, distribution_u16, seed_offset);
668 break;
669 }
670 case DataType::S16:
Georgios Pinitas21efeb42017-07-04 12:47:17 +0100671 case DataType::QS16:
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100672 {
673 ARM_COMPUTE_ERROR_ON(!(std::is_same<int16_t, D>::value));
674 std::uniform_int_distribution<int16_t> distribution_s16(low, high);
675 fill(tensor, distribution_s16, seed_offset);
676 break;
677 }
678 case DataType::U32:
679 {
680 ARM_COMPUTE_ERROR_ON(!(std::is_same<uint32_t, D>::value));
681 std::uniform_int_distribution<uint32_t> distribution_u32(low, high);
682 fill(tensor, distribution_u32, seed_offset);
683 break;
684 }
685 case DataType::S32:
686 {
687 ARM_COMPUTE_ERROR_ON(!(std::is_same<int32_t, D>::value));
688 std::uniform_int_distribution<int32_t> distribution_s32(low, high);
689 fill(tensor, distribution_s32, seed_offset);
690 break;
691 }
692 case DataType::U64:
693 {
694 ARM_COMPUTE_ERROR_ON(!(std::is_same<uint64_t, D>::value));
695 std::uniform_int_distribution<uint64_t> distribution_u64(low, high);
696 fill(tensor, distribution_u64, seed_offset);
697 break;
698 }
699 case DataType::S64:
700 {
701 ARM_COMPUTE_ERROR_ON(!(std::is_same<int64_t, D>::value));
702 std::uniform_int_distribution<int64_t> distribution_s64(low, high);
703 fill(tensor, distribution_s64, seed_offset);
704 break;
705 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100706 case DataType::F16:
707 {
Moritz Pflanzere49e2662017-07-21 15:55:28 +0100708 std::uniform_real_distribution<float> distribution_f16(low, high);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100709 fill(tensor, distribution_f16, seed_offset);
710 break;
711 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100712 case DataType::F32:
713 {
714 ARM_COMPUTE_ERROR_ON(!(std::is_same<float, D>::value));
715 std::uniform_real_distribution<float> distribution_f32(low, high);
716 fill(tensor, distribution_f32, seed_offset);
717 break;
718 }
719 case DataType::F64:
720 {
721 ARM_COMPUTE_ERROR_ON(!(std::is_same<double, D>::value));
722 std::uniform_real_distribution<double> distribution_f64(low, high);
723 fill(tensor, distribution_f64, seed_offset);
724 break;
725 }
726 case DataType::SIZET:
727 {
728 ARM_COMPUTE_ERROR_ON(!(std::is_same<size_t, D>::value));
729 std::uniform_int_distribution<size_t> distribution_sizet(low, high);
730 fill(tensor, distribution_sizet, seed_offset);
731 break;
732 }
733 default:
734 ARM_COMPUTE_ERROR("NOT SUPPORTED!");
735 }
736}
737
738template <typename T>
Moritz Pflanzerfb5aabb2017-07-18 14:39:55 +0100739void AssetsLibrary::fill_layer_data(T &&tensor, std::string name) const
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100740{
741#ifdef _WIN32
742 const std::string path_separator("\\");
Anthony Barbierac69aa12017-07-03 17:39:37 +0100743#else /* _WIN32 */
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100744 const std::string path_separator("/");
Anthony Barbierac69aa12017-07-03 17:39:37 +0100745#endif /* _WIN32 */
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100746 const std::string path = _library_path + path_separator + name;
747
SiCong Li86b53332017-08-23 11:02:43 +0100748 std::vector<unsigned long> shape;
749
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100750 // Open file
SiCong Li86b53332017-08-23 11:02:43 +0100751 std::ifstream stream(path, std::ios::in | std::ios::binary);
Anthony Barbierf6705ec2017-09-28 12:01:10 +0100752 if(!stream.good())
753 {
754 throw framework::FileNotFound("Could not load npy file: " + path);
755 }
Anthony Barbier87f21cd2017-11-10 16:27:32 +0000756 std::string header = npy::read_header(stream);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100757
SiCong Li86b53332017-08-23 11:02:43 +0100758 // Parse header
759 bool fortran_order = false;
760 std::string typestr;
Anthony Barbier87f21cd2017-11-10 16:27:32 +0000761 npy::parse_header(header, typestr, fortran_order, shape);
SiCong Li86b53332017-08-23 11:02:43 +0100762
763 // Check if the typestring matches the given one
764 std::string expect_typestr = get_typestring(tensor.data_type());
765 ARM_COMPUTE_ERROR_ON_MSG(typestr != expect_typestr, "Typestrings mismatch");
766
767 // Validate tensor shape
768 ARM_COMPUTE_ERROR_ON_MSG(shape.size() != tensor.shape().num_dimensions(), "Tensor ranks mismatch");
769 if(fortran_order)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100770 {
SiCong Li86b53332017-08-23 11:02:43 +0100771 for(size_t i = 0; i < shape.size(); ++i)
772 {
773 ARM_COMPUTE_ERROR_ON_MSG(tensor.shape()[i] != shape[i], "Tensor dimensions mismatch");
774 }
775 }
776 else
777 {
778 for(size_t i = 0; i < shape.size(); ++i)
779 {
780 ARM_COMPUTE_ERROR_ON_MSG(tensor.shape()[i] != shape[shape.size() - i - 1], "Tensor dimensions mismatch");
781 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100782 }
783
SiCong Li86b53332017-08-23 11:02:43 +0100784 // Read data
785 if(tensor.padding().empty())
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100786 {
SiCong Li86b53332017-08-23 11:02:43 +0100787 // If tensor has no padding read directly from stream.
788 stream.read(reinterpret_cast<char *>(tensor.data()), tensor.size());
789 }
790 else
791 {
792 // If tensor has padding accessing tensor elements through execution window.
793 Window window;
794 window.use_tensor_dimensions(tensor.shape());
795
SiCong Li86b53332017-08-23 11:02:43 +0100796 execute_window_loop(window, [&](const Coordinates & id)
797 {
798 stream.read(reinterpret_cast<char *>(tensor(id)), tensor.element_size());
799 });
800 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100801}
Michele Di Giorgio4d336302018-03-02 09:43:54 +0000802
803template <typename T, typename D>
804void AssetsLibrary::fill_tensor_value(T &&tensor, D value) const
805{
806 fill_tensor_uniform(tensor, 0, value, value);
807}
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100808} // namespace test
809} // namespace arm_compute
Anthony Barbierac69aa12017-07-03 17:39:37 +0100810#endif /* __ARM_COMPUTE_TEST_TENSOR_LIBRARY_H__ */