blob: 2951a16d0b8949c01edeed293740b30848dfe4cb [file] [log] [blame]
Anthony Barbier6ff3b192017-09-04 18:44:23 +01001/*
Matthew Bentham0082c362020-02-03 12:05:18 +00002 * Copyright (c) 2017-2020 ARM Limited.
Anthony Barbier6ff3b192017-09-04 18:44:23 +01003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "arm_compute/core/NEON/kernels/NETransposeKernel.h"
25
Gian Marco5420b282017-11-29 10:41:38 +000026#include "arm_compute/core/AccessWindowStatic.h"
Giorgio Arenae3d24ce2018-08-24 14:44:08 +010027#include "arm_compute/core/AccessWindowTranspose.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010028#include "arm_compute/core/Error.h"
29#include "arm_compute/core/Helpers.h"
30#include "arm_compute/core/ITensor.h"
Isabella Gottardid56e7702018-02-28 14:29:36 +000031#include "arm_compute/core/TensorInfo.h"
Gian Marco5420b282017-11-29 10:41:38 +000032#include "arm_compute/core/Utils.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010033#include "arm_compute/core/Validate.h"
34
35#include <arm_neon.h>
36
37using namespace arm_compute;
38
39namespace arm_compute
40{
41class Coordinates;
42} // namespace arm_compute
43
44namespace
45{
Gian Marco7c435f22017-12-05 16:17:23 +000046TensorShape transposed_tensor_shape(const TensorShape &in)
47{
48 TensorShape output_shape{ in };
49 const size_t w_out = in[1];
50 const size_t h_out = in[0];
51 output_shape.set(0, w_out);
52 output_shape.set(1, h_out);
53
54 return output_shape;
55}
56
Georgios Pinitas631c41a2017-12-06 11:53:03 +000057Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output)
Gian Marco7c435f22017-12-05 16:17:23 +000058{
Georgios Pinitas33843562019-12-10 13:33:18 +000059 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input);
Anthony Barbiereaefd002018-07-20 17:49:35 +010060 //Note: ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input) is not needed here as this kernel doesn't use NEON FP16 instructions.
Georgios Pinitas33843562019-12-10 13:33:18 +000061 ARM_COMPUTE_RETURN_ERROR_ON(input->data_type() == DataType::UNKNOWN);
Gian Marco7c435f22017-12-05 16:17:23 +000062
63 if(output->total_size() != 0)
64 {
65 const TensorInfo tensor_info = input->clone()->set_tensor_shape(transposed_tensor_shape(input->tensor_shape()));
66
67 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(output, &tensor_info);
68 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
Isabella Gottardi0a1090a2019-02-14 18:07:36 +000069 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_QUANTIZATION_INFO(input, output);
Gian Marco7c435f22017-12-05 16:17:23 +000070 }
71
Georgios Pinitas631c41a2017-12-06 11:53:03 +000072 return Status{};
Gian Marco7c435f22017-12-05 16:17:23 +000073}
Michalis Spyrou0b1452d2020-02-27 16:20:19 +000074unsigned int num_elems_processed(size_t element_size)
Gian Marco7c435f22017-12-05 16:17:23 +000075{
Michalis Spyrou0b1452d2020-02-27 16:20:19 +000076 switch(element_size)
Gian Marco7c435f22017-12-05 16:17:23 +000077 {
Michalis Spyrou0b1452d2020-02-27 16:20:19 +000078 case 1:
79 return 8;
80 case 2:
81 case 4:
82 return 4;
83 default:
84 break;
Gian Marco7c435f22017-12-05 16:17:23 +000085 }
86
Michalis Spyrou0b1452d2020-02-27 16:20:19 +000087 ARM_COMPUTE_ERROR("Element size not supported");
Gian Marco7c435f22017-12-05 16:17:23 +000088}
89
Anthony Barbier6ff3b192017-09-04 18:44:23 +010090void transpose_8bit_elements(const ITensor *in, ITensor *out, const Window &window)
91{
Gian Marcob42d53c2017-12-07 10:09:07 +000092 const int window_step_x = 8;
93 const int window_step_y = 8;
94 const int window_start_x = window.x().start();
95 const int window_end_x = window.x().end();
96 const int window_start_y = window.y().start();
97 const int window_end_y = std::min(window.y().end(), static_cast<int>(in->info()->dimension(1)));
98 const int window_end_y_multiple_of = ((window_end_y - window_start_y) / window_step_y) * window_step_y;
99 const size_t input_stride_in_bytes = in->info()->strides_in_bytes()[1];
100 const size_t output_stride_in_bytes = out->info()->strides_in_bytes()[1];
101
102 // Check if we need a left-over loop for the y dimension
103 bool left_over_loop_y = (((window_end_y - window_start_y) % window_step_y) != 0);
104
105 Window window_in(window);
106 window_in.set(Window::DimX, Window::Dimension(0, 1, 1));
107 if(left_over_loop_y)
108 {
109 // Check if window_end_y_multiple_of is greater than window_start_y
110 if(window_end_y_multiple_of > window_start_y)
111 {
112 window_in.set(Window::DimY, Window::Dimension(window_start_y, window_end_y_multiple_of, window_step_y));
113 }
114 else
115 {
116 window_in.set(Window::DimY, Window::Dimension(0, 0, 1));
117 }
118 }
119
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100120 Window window_out(window);
121 window_out.set(Window::DimX, Window::Dimension(0, 0, 0));
122 window_out.set(Window::DimY, Window::Dimension(0, 0, 0));
123
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100124 Iterator output(out, window_out);
125
Gian Marcob42d53c2017-12-07 10:09:07 +0000126 // Run the NEON path if and only if the input is not a row-vector
127 if(in->info()->dimension(1) != 1)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100128 {
Gian Marcob42d53c2017-12-07 10:09:07 +0000129 Iterator input(in, window_in);
130 execute_window_loop(window_in, [&](const Coordinates & id)
131 {
132 // Compute 8x8 elements per iteration
133 int x = window_start_x;
134 for(; x <= (window_end_x - window_step_x); x += window_step_x)
135 {
136 const uint8x8_t row0 = vld1_u8(reinterpret_cast<const uint8_t *>(input.ptr() + x + 0 * input_stride_in_bytes));
137 const uint8x8_t row1 = vld1_u8(reinterpret_cast<const uint8_t *>(input.ptr() + x + 1 * input_stride_in_bytes));
138 const uint8x8_t row2 = vld1_u8(reinterpret_cast<const uint8_t *>(input.ptr() + x + 2 * input_stride_in_bytes));
139 const uint8x8_t row3 = vld1_u8(reinterpret_cast<const uint8_t *>(input.ptr() + x + 3 * input_stride_in_bytes));
140 const uint8x8_t row4 = vld1_u8(reinterpret_cast<const uint8_t *>(input.ptr() + x + 4 * input_stride_in_bytes));
141 const uint8x8_t row5 = vld1_u8(reinterpret_cast<const uint8_t *>(input.ptr() + x + 5 * input_stride_in_bytes));
142 const uint8x8_t row6 = vld1_u8(reinterpret_cast<const uint8_t *>(input.ptr() + x + 6 * input_stride_in_bytes));
143 const uint8x8_t row7 = vld1_u8(reinterpret_cast<const uint8_t *>(input.ptr() + x + 7 * input_stride_in_bytes));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100144
Gian Marcob42d53c2017-12-07 10:09:07 +0000145 // Transpose 2x2
146 const uint8x8x2_t k0_u8 = vtrn_u8(row0, row1);
147 const uint8x8x2_t k1_u8 = vtrn_u8(row2, row3);
148 const uint8x8x2_t k2_u8 = vtrn_u8(row4, row5);
149 const uint8x8x2_t k3_u8 = vtrn_u8(row6, row7);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100150
Gian Marcob42d53c2017-12-07 10:09:07 +0000151 // Transpose 4x4
152 const uint16x4x2_t k0_u16 = vtrn_u16(vreinterpret_u16_u8(k0_u8.val[0]), vreinterpret_u16_u8(k1_u8.val[0]));
153 const uint16x4x2_t k1_u16 = vtrn_u16(vreinterpret_u16_u8(k0_u8.val[1]), vreinterpret_u16_u8(k1_u8.val[1]));
154 const uint16x4x2_t k2_u16 = vtrn_u16(vreinterpret_u16_u8(k2_u8.val[0]), vreinterpret_u16_u8(k3_u8.val[0]));
155 const uint16x4x2_t k3_u16 = vtrn_u16(vreinterpret_u16_u8(k2_u8.val[1]), vreinterpret_u16_u8(k3_u8.val[1]));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100156
Gian Marcob42d53c2017-12-07 10:09:07 +0000157 // Transpose 8x8
158 const uint32x2x2_t k0_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[0]), vreinterpret_u32_u16(k2_u16.val[0]));
159 const uint32x2x2_t k1_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[1]), vreinterpret_u32_u16(k2_u16.val[1]));
160 const uint32x2x2_t k2_u32 = vtrn_u32(vreinterpret_u32_u16(k1_u16.val[0]), vreinterpret_u32_u16(k3_u16.val[0]));
161 const uint32x2x2_t k3_u32 = vtrn_u32(vreinterpret_u32_u16(k1_u16.val[1]), vreinterpret_u32_u16(k3_u16.val[1]));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100162
Gian Marcob42d53c2017-12-07 10:09:07 +0000163 // Compute destination address
164 const size_t dst_offset_in_bytes = id.y() * sizeof(uint8_t) + x * output_stride_in_bytes;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100165
Gian Marcob42d53c2017-12-07 10:09:07 +0000166 vst1_u8(reinterpret_cast<uint8_t *>(output.ptr() + dst_offset_in_bytes + 0 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k0_u32.val[0])));
167 vst1_u8(reinterpret_cast<uint8_t *>(output.ptr() + dst_offset_in_bytes + 1 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k2_u32.val[0])));
168 vst1_u8(reinterpret_cast<uint8_t *>(output.ptr() + dst_offset_in_bytes + 2 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k1_u32.val[0])));
169 vst1_u8(reinterpret_cast<uint8_t *>(output.ptr() + dst_offset_in_bytes + 3 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k3_u32.val[0])));
170 vst1_u8(reinterpret_cast<uint8_t *>(output.ptr() + dst_offset_in_bytes + 4 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k0_u32.val[1])));
171 vst1_u8(reinterpret_cast<uint8_t *>(output.ptr() + dst_offset_in_bytes + 5 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k2_u32.val[1])));
172 vst1_u8(reinterpret_cast<uint8_t *>(output.ptr() + dst_offset_in_bytes + 6 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k1_u32.val[1])));
173 vst1_u8(reinterpret_cast<uint8_t *>(output.ptr() + dst_offset_in_bytes + 7 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k3_u32.val[1])));
174 }
175
176 // Compute left-over elements along the x dimension (1x8)
177 for(; x < window_end_x; ++x)
178 {
179 const uint8_t val0 = *(input.ptr() + x + 0 * input_stride_in_bytes);
180 const uint8_t val1 = *(input.ptr() + x + 1 * input_stride_in_bytes);
181 const uint8_t val2 = *(input.ptr() + x + 2 * input_stride_in_bytes);
182 const uint8_t val3 = *(input.ptr() + x + 3 * input_stride_in_bytes);
183 const uint8_t val4 = *(input.ptr() + x + 4 * input_stride_in_bytes);
184 const uint8_t val5 = *(input.ptr() + x + 5 * input_stride_in_bytes);
185 const uint8_t val6 = *(input.ptr() + x + 6 * input_stride_in_bytes);
186 const uint8_t val7 = *(input.ptr() + x + 7 * input_stride_in_bytes);
187
188 uint8x8_t result = vdup_n_u8(0);
189 result = vset_lane_u8(val0, result, 0);
190 result = vset_lane_u8(val1, result, 1);
191 result = vset_lane_u8(val2, result, 2);
192 result = vset_lane_u8(val3, result, 3);
193 result = vset_lane_u8(val4, result, 4);
194 result = vset_lane_u8(val5, result, 5);
195 result = vset_lane_u8(val6, result, 6);
196 result = vset_lane_u8(val7, result, 7);
197
198 // Compute destination address
199 const size_t dst_offset_in_bytes = id.y() * sizeof(uint8_t) + x * output_stride_in_bytes;
200
201 vst1_u8(output.ptr() + dst_offset_in_bytes, result);
202 }
203 },
204 input, output);
205 }
206
207 if(left_over_loop_y)
208 {
209 window_in.set(Window::DimX, Window::Dimension(window.x().start(), window.x().end(), 1));
210 window_in.set(Window::DimY, Window::Dimension(window_end_y_multiple_of, window_end_y, 1));
211
212 Iterator input(in, window_in);
213 Iterator output(out, window_out);
214
215 // Compute left-over elements along the y dimension (1x1)
216 execute_window_loop(window_in, [&](const Coordinates & id)
217 {
218 const uint8_t val0 = *input.ptr();
219
220 // Compute destination address
221 const size_t dst_offset_in_bytes = id.y() * sizeof(uint8_t) + id.x() * output_stride_in_bytes;
222
223 *(output.ptr() + dst_offset_in_bytes) = val0;
224 },
225 input, output);
226 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100227}
228
229void transpose_16bit_elements(const ITensor *in, ITensor *out, const Window &window)
230{
Gian Marcob42d53c2017-12-07 10:09:07 +0000231 const int window_step_x = 4;
232 const int window_step_y = 4;
233 const int window_start_x = window.x().start();
234 const int window_end_x = window.x().end();
235 const int window_start_y = window.y().start();
236 const int window_end_y = std::min(window.y().end(), static_cast<int>(in->info()->dimension(1)));
237 const int window_end_y_multiple_of = ((window_end_y - window_start_y) / window_step_y) * window_step_y;
238 const size_t input_stride_in_bytes = in->info()->strides_in_bytes()[1];
239 const size_t output_stride_in_bytes = out->info()->strides_in_bytes()[1];
240
241 // Check if we need a left-over loop for the y dimension
242 bool left_over_loop_y = (((window_end_y - window_start_y) % window_step_y) != 0);
243
244 Window window_in(window);
245 window_in.set(Window::DimX, Window::Dimension(0, 1, 1));
246 if(left_over_loop_y)
247 {
248 // Check if window_end_y_multiple_of is greater than window_start_y
249 if(window_end_y_multiple_of > window_start_y)
250 {
251 window_in.set(Window::DimY, Window::Dimension(window_start_y, window_end_y_multiple_of, window_step_y));
252 }
253 else
254 {
255 window_in.set(Window::DimY, Window::Dimension(0, 0, 1));
256 }
257 }
258
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100259 Window window_out(window);
260 window_out.set(Window::DimX, Window::Dimension(0, 0, 0));
261 window_out.set(Window::DimY, Window::Dimension(0, 0, 0));
262
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100263 Iterator output(out, window_out);
264
Gian Marcob42d53c2017-12-07 10:09:07 +0000265 // Run the NEON path if and only if the input is not a row-vector
266 if(in->info()->dimension(1) != 1)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100267 {
Gian Marcob42d53c2017-12-07 10:09:07 +0000268 Iterator input(in, window_in);
269 execute_window_loop(window_in, [&](const Coordinates & id)
270 {
271 // Compute 4x4 elements per iteration
272 int x = window_start_x;
273 for(; x <= (window_end_x - window_step_x); x += window_step_x)
274 {
275 const uint16x4_t row0 = vld1_u16(reinterpret_cast<const uint16_t *>(input.ptr() + 0 * input_stride_in_bytes) + x);
276 const uint16x4_t row1 = vld1_u16(reinterpret_cast<const uint16_t *>(input.ptr() + 1 * input_stride_in_bytes) + x);
277 const uint16x4_t row2 = vld1_u16(reinterpret_cast<const uint16_t *>(input.ptr() + 2 * input_stride_in_bytes) + x);
278 const uint16x4_t row3 = vld1_u16(reinterpret_cast<const uint16_t *>(input.ptr() + 3 * input_stride_in_bytes) + x);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100279
Gian Marcob42d53c2017-12-07 10:09:07 +0000280 // Transpose 2x2
281 const uint16x4x2_t k0_u16 = vtrn_u16(row0, row1);
282 const uint16x4x2_t k1_u16 = vtrn_u16(row2, row3);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100283
Gian Marcob42d53c2017-12-07 10:09:07 +0000284 // Transpose 4x4
285 const uint32x2x2_t k0_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[0]), vreinterpret_u32_u16(k1_u16.val[0]));
286 const uint32x2x2_t k1_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[1]), vreinterpret_u32_u16(k1_u16.val[1]));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100287
Gian Marcob42d53c2017-12-07 10:09:07 +0000288 // Compute destination address
289 const size_t dst_offset_in_bytes = id.y() * sizeof(uint16_t) + x * output_stride_in_bytes;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100290
Gian Marcob42d53c2017-12-07 10:09:07 +0000291 vst1_u16(reinterpret_cast<uint16_t *>(output.ptr() + dst_offset_in_bytes + 0 * output_stride_in_bytes), vreinterpret_u16_u32(k0_u32.val[0]));
292 vst1_u16(reinterpret_cast<uint16_t *>(output.ptr() + dst_offset_in_bytes + 1 * output_stride_in_bytes), vreinterpret_u16_u32(k1_u32.val[0]));
293 vst1_u16(reinterpret_cast<uint16_t *>(output.ptr() + dst_offset_in_bytes + 2 * output_stride_in_bytes), vreinterpret_u16_u32(k0_u32.val[1]));
294 vst1_u16(reinterpret_cast<uint16_t *>(output.ptr() + dst_offset_in_bytes + 3 * output_stride_in_bytes), vreinterpret_u16_u32(k1_u32.val[1]));
295 }
296
297 // Compute left-over elements (1x4)
298 for(; x < window_end_x; ++x)
299 {
300 const uint16_t val0 = *(reinterpret_cast<uint16_t *>(input.ptr() + 0 * input_stride_in_bytes) + x);
301 const uint16_t val1 = *(reinterpret_cast<uint16_t *>(input.ptr() + 1 * input_stride_in_bytes) + x);
302 const uint16_t val2 = *(reinterpret_cast<uint16_t *>(input.ptr() + 2 * input_stride_in_bytes) + x);
303 const uint16_t val3 = *(reinterpret_cast<uint16_t *>(input.ptr() + 3 * input_stride_in_bytes) + x);
304
305 uint16x4_t result = vdup_n_u16(0);
306 result = vset_lane_u16(val0, result, 0);
307 result = vset_lane_u16(val1, result, 1);
308 result = vset_lane_u16(val2, result, 2);
309 result = vset_lane_u16(val3, result, 3);
310
311 // Compute destination address
312 const size_t dst_offset_in_bytes = id.y() * sizeof(uint16_t) + x * output_stride_in_bytes;
313
314 vst1_u16(reinterpret_cast<uint16_t *>(output.ptr() + dst_offset_in_bytes), result);
315 }
316 },
317 input, output);
318 }
319
320 if(left_over_loop_y)
321 {
322 window_in.set(Window::DimX, Window::Dimension(window.x().start(), window.x().end(), 1));
323 window_in.set(Window::DimY, Window::Dimension(window_end_y_multiple_of, window_end_y, 1));
324
325 Iterator input(in, window_in);
326 Iterator output(out, window_out);
327
328 // Compute left-over elements along the y dimension (1x1)
329 execute_window_loop(window_in, [&](const Coordinates & id)
330 {
331 const uint16_t val0 = *(reinterpret_cast<uint16_t *>(input.ptr()));
332
333 // Compute destination address
334 const size_t dst_offset_in_bytes = id.y() * sizeof(uint16_t) + id.x() * output_stride_in_bytes;
335
336 *(reinterpret_cast<uint16_t *>(output.ptr() + dst_offset_in_bytes)) = val0;
337 },
338 input, output);
339 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100340}
341
342void transpose_32bit_elements(const ITensor *in, ITensor *out, const Window &window)
343{
Gian Marcob42d53c2017-12-07 10:09:07 +0000344 const int window_step_x = 4;
345 const int window_step_y = 4;
346 const int window_start_x = window.x().start();
347 const int window_end_x = window.x().end();
348 const int window_start_y = window.y().start();
349 const int window_end_y = std::min(window.y().end(), static_cast<int>(in->info()->dimension(1)));
350 const int window_end_y_multiple_of = ((window_end_y - window_start_y) / window_step_y) * window_step_y;
351 const size_t input_stride_in_bytes = in->info()->strides_in_bytes()[1];
352 const size_t output_stride_in_bytes = out->info()->strides_in_bytes()[1];
353
354 // Check if we need a left-over loop for the y dimension
355 bool left_over_loop_y = (((window_end_y - window_start_y) % window_step_y) != 0);
356
357 Window window_in(window);
358 window_in.set(Window::DimX, Window::Dimension(0, 1, 1));
359 if(left_over_loop_y)
360 {
361 // Check if window_end_y_multiple_of is greater than window_start_y
362 if(window_end_y_multiple_of > window_start_y)
363 {
364 window_in.set(Window::DimY, Window::Dimension(window_start_y, window_end_y_multiple_of, window_step_y));
365 }
366 else
367 {
368 window_in.set(Window::DimY, Window::Dimension(0, 0, 1));
369 }
370 }
371
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100372 Window window_out(window);
373 window_out.set(Window::DimX, Window::Dimension(0, 0, 0));
374 window_out.set(Window::DimY, Window::Dimension(0, 0, 0));
375
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100376 Iterator output(out, window_out);
377
Gian Marcob42d53c2017-12-07 10:09:07 +0000378 // Run the NEON path if and only if the input is not a row-vector
379 if(in->info()->dimension(1) != 1)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100380 {
Gian Marcob42d53c2017-12-07 10:09:07 +0000381 Iterator input(in, window_in);
382 execute_window_loop(window_in, [&](const Coordinates & id)
383 {
384 // Compute 4x4 elements per iteration
385 int x = window_start_x;
386 for(; x <= (window_end_x - window_step_x); x += window_step_x)
387 {
388 const uint32x4_t row0 = vld1q_u32(reinterpret_cast<const uint32_t *>(input.ptr() + 0 * input_stride_in_bytes) + x);
389 const uint32x4_t row1 = vld1q_u32(reinterpret_cast<const uint32_t *>(input.ptr() + 1 * input_stride_in_bytes) + x);
390 const uint32x4_t row2 = vld1q_u32(reinterpret_cast<const uint32_t *>(input.ptr() + 2 * input_stride_in_bytes) + x);
391 const uint32x4_t row3 = vld1q_u32(reinterpret_cast<const uint32_t *>(input.ptr() + 3 * input_stride_in_bytes) + x);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100392
Gian Marcob42d53c2017-12-07 10:09:07 +0000393 // Transpose 2x2
394 const uint32x2x2_t k0_u32 = vtrn_u32(vget_low_u32(row0), vget_low_u32(row1));
395 const uint32x2x2_t k1_u32 = vtrn_u32(vget_high_u32(row2), vget_high_u32(row3));
396 const uint32x2x2_t k2_u32 = vtrn_u32(vget_high_u32(row0), vget_high_u32(row1));
397 const uint32x2x2_t k3_u32 = vtrn_u32(vget_low_u32(row2), vget_low_u32(row3));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100398
Gian Marcob42d53c2017-12-07 10:09:07 +0000399 // Compute destination address
400 const size_t dst_offset_in_bytes = id.y() * sizeof(uint32_t) + x * output_stride_in_bytes;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100401
Gian Marcob42d53c2017-12-07 10:09:07 +0000402 // Swap block 01 with block 10 and store
403 vst1q_u32(reinterpret_cast<uint32_t *>(output.ptr() + dst_offset_in_bytes + 0 * output_stride_in_bytes), vcombine_u32(k0_u32.val[0], k3_u32.val[0]));
404 vst1q_u32(reinterpret_cast<uint32_t *>(output.ptr() + dst_offset_in_bytes + 1 * output_stride_in_bytes), vcombine_u32(k0_u32.val[1], k3_u32.val[1]));
405 vst1q_u32(reinterpret_cast<uint32_t *>(output.ptr() + dst_offset_in_bytes + 2 * output_stride_in_bytes), vcombine_u32(k2_u32.val[0], k1_u32.val[0]));
406 vst1q_u32(reinterpret_cast<uint32_t *>(output.ptr() + dst_offset_in_bytes + 3 * output_stride_in_bytes), vcombine_u32(k2_u32.val[1], k1_u32.val[1]));
407 }
408
409 // Compute left-over elements (1x4)
410 for(; x < window_end_x; ++x)
411 {
412 const uint32_t val0 = *(reinterpret_cast<uint32_t *>(input.ptr() + 0 * input_stride_in_bytes) + x);
413 const uint32_t val1 = *(reinterpret_cast<uint32_t *>(input.ptr() + 1 * input_stride_in_bytes) + x);
414 const uint32_t val2 = *(reinterpret_cast<uint32_t *>(input.ptr() + 2 * input_stride_in_bytes) + x);
415 const uint32_t val3 = *(reinterpret_cast<uint32_t *>(input.ptr() + 3 * input_stride_in_bytes) + x);
416
417 uint32x4_t result = vdupq_n_u32(0);
418 result = vsetq_lane_u32(val0, result, 0);
419 result = vsetq_lane_u32(val1, result, 1);
420 result = vsetq_lane_u32(val2, result, 2);
421 result = vsetq_lane_u32(val3, result, 3);
422
423 // Compute destination address
424 const size_t dst_offset_in_bytes = id.y() * sizeof(uint32_t) + x * output_stride_in_bytes;
425
426 vst1q_u32(reinterpret_cast<uint32_t *>(output.ptr() + dst_offset_in_bytes), result);
427 }
428 },
429 input, output);
430 }
431
432 if(left_over_loop_y)
433 {
434 window_in.set(Window::DimX, Window::Dimension(window.x().start(), window.x().end(), 1));
435 window_in.set(Window::DimY, Window::Dimension(window_end_y_multiple_of, window_end_y, 1));
436
437 Iterator input(in, window_in);
438 Iterator output(out, window_out);
439
440 // Compute left-over elements along the y dimension (1x1)
441 execute_window_loop(window_in, [&](const Coordinates & id)
442 {
443 const uint32_t val0 = *(reinterpret_cast<uint32_t *>(input.ptr()));
444
445 // Compute destination address
446 const size_t dst_offset_in_bytes = id.y() * sizeof(uint32_t) + id.x() * output_stride_in_bytes;
447
448 *(reinterpret_cast<uint32_t *>(output.ptr() + dst_offset_in_bytes)) = val0;
449 },
450 input, output);
451 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100452}
453} // namespace
454
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000455Status NETransposeKernel::validate(const ITensorInfo *input, const ITensorInfo *output)
Gian Marco7c435f22017-12-05 16:17:23 +0000456{
457 ARM_COMPUTE_ERROR_ON_NULLPTR(input, output);
458 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, output));
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000459 return Status{};
Gian Marco7c435f22017-12-05 16:17:23 +0000460}
461
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100462NETransposeKernel::NETransposeKernel()
463 : _func(nullptr), _input(nullptr), _output(nullptr)
464{
465}
466
467void NETransposeKernel::configure(const ITensor *input, ITensor *output)
468{
Gian Marco7c435f22017-12-05 16:17:23 +0000469 ARM_COMPUTE_ERROR_ON_NULLPTR(input, output);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100470
471 // Output tensor auto inizialitation if not yet initialized
Gian Marco7c435f22017-12-05 16:17:23 +0000472 auto_init_if_empty(*output->info(), input->info()->clone()->set_tensor_shape(transposed_tensor_shape(input->info()->tensor_shape())));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100473
Gian Marco7c435f22017-12-05 16:17:23 +0000474 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), output->info()));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100475
476 _input = input;
477 _output = output;
478
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100479 switch(input->info()->element_size())
480 {
481 case 1:
Gian Marco7c435f22017-12-05 16:17:23 +0000482 _func = &transpose_8bit_elements;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100483 break;
484 case 2:
Gian Marco7c435f22017-12-05 16:17:23 +0000485 _func = &transpose_16bit_elements;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100486 break;
487 case 4:
Gian Marco7c435f22017-12-05 16:17:23 +0000488 _func = &transpose_32bit_elements;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100489 break;
490 default:
491 ARM_COMPUTE_ERROR("Element size not supported");
492 break;
493 }
494
495 // Configure kernel window
Michalis Spyrou0b1452d2020-02-27 16:20:19 +0000496 Coordinates coord;
497 coord.set_num_dimensions(output->info()->num_dimensions());
498 output->info()->set_valid_region(ValidRegion(coord, output->info()->tensor_shape()));
499
500 // Note: This kernel performs 16 elements per iteration.
501 // However, since we use a left-over for loop on both dimensions (X and Y), we cannot have any read or write out of memory
502 // For this reason num_elems_processed_per_iteration_x is set to 1
503 const unsigned int num_elems_processed_per_iteration_x = 1;
504 const unsigned int num_elems_processed_per_iteration_y = num_elems_processed(input->info()->element_size());
505
506 // Configure kernel window
507 Window win = calculate_max_window(*input->info(), Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));
508
509 INEKernel::configure(win);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100510}
511
Moritz Pflanzerc186b572017-09-07 09:48:04 +0100512void NETransposeKernel::run(const Window &window, const ThreadInfo &info)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100513{
Moritz Pflanzerc186b572017-09-07 09:48:04 +0100514 ARM_COMPUTE_UNUSED(info);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100515 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
516 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window);
517 ARM_COMPUTE_ERROR_ON(_func == nullptr);
518
519 (*_func)(_input, _output, window);
520}