blob: 908fa13876247f5f4a530696c9cbfe0c5b42f0d9 [file] [log] [blame]
Michalis Spyrou7362f0d2017-10-18 17:58:22 +01001/*
Georgios Pinitasf72f9362018-01-12 16:29:45 +00002 * Copyright (c) 2017-2018 ARM Limited.
Michalis Spyrou7362f0d2017-10-18 17:58:22 +01003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25#ifndef __ARM_COMPUTE_NEDIRECTCONVOLUTIONDETAIL_H__
26#define __ARM_COMPUTE_NEDIRECTCONVOLUTIONDETAIL_H__
27
28#include "arm_compute/core/AccessWindowStatic.h"
29#include "arm_compute/core/NEON/NEFixedPoint.h"
30
31#include <arm_neon.h>
32
33namespace arm_compute
34{
35namespace detail
36{
37/** Loads a 3x3 matrix as a row (float).
38 *
Georgios Pinitasf72f9362018-01-12 16:29:45 +000039 * @param[in] ptr Pointer to a float 3x3 matrix.
40 * @param[in] weights_offset (Optional) Weights quantization offset.
Michalis Spyrou7362f0d2017-10-18 17:58:22 +010041 *
42 * @return The loaded matrix.
43 */
Georgios Pinitasf72f9362018-01-12 16:29:45 +000044inline float32x4x3_t load_matrix_row(const float *ptr, int weights_offset = 0)
Michalis Spyrou7362f0d2017-10-18 17:58:22 +010045{
Georgios Pinitasf72f9362018-01-12 16:29:45 +000046 ARM_COMPUTE_UNUSED(weights_offset);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +010047 const float32x4x3_t r =
48 {
49 {
50 vld1q_dup_f32(ptr),
51 vld1q_dup_f32(1 + ptr),
52 vld1q_dup_f32(2 + ptr)
53 }
54 };
55 return r;
56}
57
58/** Loads a 3x3 matrix as a row (qint8_t).
59 *
Georgios Pinitasf72f9362018-01-12 16:29:45 +000060 * @param[in] ptr Pointer to a qint8 3x3 matrix.
61 * @param[in] weights_offset (Optional) Weights quantization offset.
Michalis Spyrou7362f0d2017-10-18 17:58:22 +010062 *
63 * @return The loaded matrix.
64 */
Georgios Pinitasf72f9362018-01-12 16:29:45 +000065inline qint8x8x3_t load_matrix_row(const qint8_t *ptr, int weights_offset = 0)
Michalis Spyrou7362f0d2017-10-18 17:58:22 +010066{
Georgios Pinitasf72f9362018-01-12 16:29:45 +000067 ARM_COMPUTE_UNUSED(weights_offset);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +010068 /* ptr is a pointer to a row in a 3x3 matrix, the function returns 3 vectors holding exactly the same value in all lanes:
69 r.val[0] contains the first element, r.val[1] the second element and r.val[2] the third element (in all lanes) */
70 const qint8x8x3_t r =
71 {
72 {
73 vld1_dup_qs8(ptr),
74 vld1_dup_qs8(1 + ptr),
75 vld1_dup_qs8(2 + ptr)
76 }
77 };
78 return r;
79}
80
Georgios Pinitasf72f9362018-01-12 16:29:45 +000081/** Loads a 3x3 matrix as a row (uint8_t).
82 *
83 * @param[in] ptr Pointer to a uint8_t 3x3 matrix.
84 * @param[in] weights_offset (Optional) Weights quantization offset.
85 *
86 * @return The loaded matrix.
87 */
88inline int32x4x3_t load_matrix_row(const uint8_t *ptr, int weights_offset = 0)
89{
90 const int32x4_t v_weights_offset = vdupq_n_s32(weights_offset);
91
92 /* ptr is a pointer to a row in a 3x3 matrix, the function returns 3 vectors holding exactly the same value in all lanes:
93 r.val[0] contains the first element, r.val[1] the second element and r.val[2] the third element (in all lanes) */
94 int32x4x3_t r =
95 {
96 {
97 vaddq_s32(v_weights_offset, vdupq_n_s32(*ptr)),
98 vaddq_s32(v_weights_offset, vdupq_n_s32(*(ptr + 1))),
99 vaddq_s32(v_weights_offset, vdupq_n_s32(*(ptr + 2)))
100 }
101 };
102 return r;
103}
104
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100105/** Perform a convolve3x3 on float32.
106 *
107 * @param[in] in_top Pointer to the first row of the input.
108 * @param[in] in_mid Pointer to the second row of the input.
109 * @param[in] in_low Pointer to the third row of the input.
110 * @param[in] m0 First row of the filter.
111 * @param[in] m1 Second row of the filter.
112 * @param[in] m2 Third row of the filter.
113 * @param[in] fixed_point_position (Optional) Fixed point position.
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000114 * @param[in] input_offset (Optional) Input quantization offset.
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100115 *
116 */
117template <unsigned int stridex>
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000118float32x4x2_t convolve_3x3(const float *in_top, const float *in_mid, const float *in_low,
119 const float32x4x3_t &m0, const float32x4x3_t &m1, const float32x4x3_t &m2,
120 int fixed_point_position, int input_offset = 0);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100121
122template <>
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000123inline float32x4x2_t convolve_3x3<1>(const float *in_top, const float *in_mid, const float *in_low,
124 const float32x4x3_t &m0, const float32x4x3_t &m1, const float32x4x3_t &m2,
125 int fixed_point_position, int input_offset)
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100126{
127 ARM_COMPUTE_UNUSED(fixed_point_position);
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000128 ARM_COMPUTE_UNUSED(input_offset);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100129
130 const float32x4x3_t vtop =
131 {
132 {
133 vld1q_f32(in_top),
134 vld1q_f32(in_top + 4),
135 vld1q_f32(in_top + 8)
136 }
137 };
138 const float32x4x3_t vmid =
139 {
140 {
141 vld1q_f32(in_mid),
142 vld1q_f32(in_mid + 4),
143 vld1q_f32(in_mid + 8)
144 }
145 };
146 const float32x4x3_t vlow =
147 {
148 {
149 vld1q_f32(in_low),
150 vld1q_f32(in_low + 4),
151 vld1q_f32(in_low + 8)
152 }
153 };
154 float32x4x2_t out =
155 {
156 {
157 vmulq_f32(vtop.val[0], m0.val[0]),
158 vmulq_f32(vtop.val[1], m0.val[0])
159 }
160 };
161 out.val[0] = vmlaq_f32(out.val[0], vextq_f32(vtop.val[0], vtop.val[1], 1), m0.val[1]);
162 out.val[0] = vmlaq_f32(out.val[0], vextq_f32(vtop.val[0], vtop.val[1], 2), m0.val[2]);
163
164 out.val[0] = vmlaq_f32(out.val[0], vmid.val[0], m1.val[0]);
165 out.val[0] = vmlaq_f32(out.val[0], vextq_f32(vmid.val[0], vmid.val[1], 1), m1.val[1]);
166 out.val[0] = vmlaq_f32(out.val[0], vextq_f32(vmid.val[0], vmid.val[1], 2), m1.val[2]);
167
168 out.val[0] = vmlaq_f32(out.val[0], vlow.val[0], m2.val[0]);
169 out.val[0] = vmlaq_f32(out.val[0], vextq_f32(vlow.val[0], vlow.val[1], 1), m2.val[1]);
170 out.val[0] = vmlaq_f32(out.val[0], vextq_f32(vlow.val[0], vlow.val[1], 2), m2.val[2]);
171
172 out.val[1] = vmlaq_f32(out.val[1], vextq_f32(vtop.val[1], vtop.val[2], 1), m0.val[1]);
173 out.val[1] = vmlaq_f32(out.val[1], vextq_f32(vtop.val[1], vtop.val[2], 2), m0.val[2]);
174
175 out.val[1] = vmlaq_f32(out.val[1], vmid.val[1], m1.val[0]);
176 out.val[1] = vmlaq_f32(out.val[1], vextq_f32(vmid.val[1], vmid.val[2], 1), m1.val[1]);
177 out.val[1] = vmlaq_f32(out.val[1], vextq_f32(vmid.val[1], vmid.val[2], 2), m1.val[2]);
178
179 out.val[1] = vmlaq_f32(out.val[1], vlow.val[1], m2.val[0]);
180 out.val[1] = vmlaq_f32(out.val[1], vextq_f32(vlow.val[1], vlow.val[2], 1), m2.val[1]);
181 out.val[1] = vmlaq_f32(out.val[1], vextq_f32(vlow.val[1], vlow.val[2], 2), m2.val[2]);
182 return out;
183}
184
185template <>
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000186inline float32x4x2_t convolve_3x3<2>(const float *in_top, const float *in_mid, const float *in_low,
187 const float32x4x3_t &m0, const float32x4x3_t &m1, const float32x4x3_t &m2,
188 int fixed_point_position, int input_offset)
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100189{
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000190 ARM_COMPUTE_UNUSED(input_offset);
191
192 float32x4x2_t out = convolve_3x3<1>(in_top, in_mid, in_low, m0, m1, m2, fixed_point_position, input_offset);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100193 out.val[0] = vsetq_lane_f32(vgetq_lane_f32(out.val[0], 2), out.val[0], 1);
194 out.val[0] = vsetq_lane_f32(vgetq_lane_f32(out.val[1], 0), out.val[0], 2);
195 out.val[0] = vsetq_lane_f32(vgetq_lane_f32(out.val[1], 2), out.val[0], 3);
196 return out;
197}
198
199template <>
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000200inline float32x4x2_t convolve_3x3<3>(const float *in_top, const float *in_mid, const float *in_low,
201 const float32x4x3_t &m0, const float32x4x3_t &m1, const float32x4x3_t &m2,
202 int fixed_point_position, int input_offset)
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100203{
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000204 ARM_COMPUTE_UNUSED(input_offset);
205
206 float32x4x2_t out = convolve_3x3<1>(in_top, in_mid, in_low, m0, m1, m2, fixed_point_position, input_offset);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100207 out.val[0] = vsetq_lane_f32(vgetq_lane_f32(out.val[0], 3), out.val[0], 1);
208 return out;
209}
210
211/** Perform a convolve3x3 on qint16.
212 *
213 * @param[in] in_top Pointer to the first row of the input.
214 * @param[in] in_mid Pointer to the second row of the input.
215 * @param[in] in_low Pointer to the third row of the input.
216 * @param[in] m0 First row of the filter.
217 * @param[in] m1 Second row of the filter.
218 * @param[in] m2 Third row of the filter.
219 * @param[in] fixed_point_position (Optional) Fixed point position.
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000220 * @param[in] input_offset (Optional) Input quantization offset.
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100221 *
222 */
223template <unsigned int stridex>
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000224qint16x8x2_t convolve_3x3(const qint8_t *in_top, const qint8_t *in_mid, const qint8_t *in_low,
225 const qint8x8x3_t &m0, const qint8x8x3_t &m1, const qint8x8x3_t &m2,
226 int fixed_point_position, int input_offset = 0);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100227
228template <>
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000229inline qint16x8x2_t convolve_3x3<1>(const qint8_t *in_top, const qint8_t *in_mid, const qint8_t *in_low,
230 const qint8x8x3_t &m0, const qint8x8x3_t &m1, const qint8x8x3_t &m2,
231 int fixed_point_position, int input_offset)
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100232{
233 ARM_COMPUTE_UNUSED(fixed_point_position);
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000234 ARM_COMPUTE_UNUSED(input_offset);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100235
236 const qint8x8x3_t vtop =
237 {
238 {
239 vld1_qs8(in_top),
240 vld1_qs8(in_top + 8),
241 vld1_qs8(in_top + 16)
242 }
243 };
244 const qint8x8x3_t vmid =
245 {
246 {
247 vld1_qs8(in_mid),
248 vld1_qs8(in_mid + 8),
249 vld1_qs8(in_mid + 16)
250 }
251 };
252 const qint8x8x3_t vlow =
253 {
254 {
255 vld1_qs8(in_low),
256 vld1_qs8(in_low + 8),
257 vld1_qs8(in_low + 16)
258 }
259 };
260 qint16x8x2_t out =
261 {
262 {
263 vmull_qs8(vtop.val[0], m0.val[0], fixed_point_position),
264 vmull_qs8(vtop.val[1], m0.val[0], fixed_point_position)
265 }
266 };
267 out.val[0] = vqmlal_qs8(out.val[0], vext_s8(vtop.val[0], vtop.val[1], 1), m0.val[1], fixed_point_position);
268 out.val[0] = vqmlal_qs8(out.val[0], vext_s8(vtop.val[0], vtop.val[1], 2), m0.val[2], fixed_point_position);
269 out.val[0] = vqmlal_qs8(out.val[0], vmid.val[0], m1.val[0], fixed_point_position);
270 out.val[0] = vqmlal_qs8(out.val[0], vext_s8(vmid.val[0], vmid.val[1], 1), m1.val[1], fixed_point_position);
271 out.val[0] = vqmlal_qs8(out.val[0], vext_s8(vmid.val[0], vmid.val[1], 2), m1.val[2], fixed_point_position);
272 out.val[0] = vqmlal_qs8(out.val[0], vlow.val[0], m2.val[0], fixed_point_position);
273 out.val[0] = vqmlal_qs8(out.val[0], vext_s8(vlow.val[0], vlow.val[1], 1), m2.val[1], fixed_point_position);
274 out.val[0] = vqmlal_qs8(out.val[0], vext_s8(vlow.val[0], vlow.val[1], 2), m2.val[2], fixed_point_position);
275 out.val[1] = vqmlal_qs8(out.val[1], vext_s8(vtop.val[1], vtop.val[2], 1), m0.val[1], fixed_point_position);
276 out.val[1] = vqmlal_qs8(out.val[1], vext_s8(vtop.val[1], vtop.val[2], 2), m0.val[2], fixed_point_position);
277 out.val[1] = vqmlal_qs8(out.val[1], vmid.val[1], m1.val[0], fixed_point_position);
278 out.val[1] = vqmlal_qs8(out.val[1], vext_s8(vmid.val[1], vmid.val[2], 1), m1.val[1], fixed_point_position);
279 out.val[1] = vqmlal_qs8(out.val[1], vext_s8(vmid.val[1], vmid.val[2], 2), m1.val[2], fixed_point_position);
280 out.val[1] = vqmlal_qs8(out.val[1], vlow.val[1], m2.val[0], fixed_point_position);
281 out.val[1] = vqmlal_qs8(out.val[1], vext_s8(vlow.val[1], vlow.val[2], 1), m2.val[1], fixed_point_position);
282 out.val[1] = vqmlal_qs8(out.val[1], vext_s8(vlow.val[1], vlow.val[2], 2), m2.val[2], fixed_point_position);
283 return out;
284}
285
286template <>
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000287inline qint16x8x2_t convolve_3x3<2>(const qint8_t *in_top, const qint8_t *in_mid, const qint8_t *in_low,
288 const qint8x8x3_t &m0, const qint8x8x3_t &m1, const qint8x8x3_t &m2,
289 int fixed_point_position, int input_offset)
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100290{
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000291 ARM_COMPUTE_UNUSED(input_offset);
292
293 qint16x8x2_t out = convolve_3x3<1>(in_top, in_mid, in_low, m0, m1, m2, fixed_point_position, input_offset);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100294 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[0], 2), out.val[0], 1);
295 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[0], 4), out.val[0], 2);
296 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[0], 6), out.val[0], 3);
297 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[1], 0), out.val[0], 4);
298 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[1], 2), out.val[0], 5);
299 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[1], 4), out.val[0], 6);
300 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[1], 6), out.val[0], 7);
301 return out;
302}
303
304template <>
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000305inline qint16x8x2_t convolve_3x3<3>(const qint8_t *in_top, const qint8_t *in_mid, const qint8_t *in_low,
306 const qint8x8x3_t &m0, const qint8x8x3_t &m1, const qint8x8x3_t &m2,
307 int fixed_point_position, int input_offset)
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100308{
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000309 ARM_COMPUTE_UNUSED(input_offset);
310
311 qint16x8x2_t out = convolve_3x3<1>(in_top, in_mid, in_low, m0, m1, m2, fixed_point_position, input_offset);
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100312 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[0], 3), out.val[0], 1);
313 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[0], 6), out.val[0], 2);
314 out.val[0] = vsetq_lane_s16(vgetq_lane_s16(out.val[1], 1), out.val[0], 3);
315 return out;
316}
317
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000318/** Perform a convolve3x3 on uint8_t
319 *
320 * @param[in] in_top Pointer to the first row of the input.
321 * @param[in] in_mid Pointer to the second row of the input.
322 * @param[in] in_low Pointer to the third row of the input.
323 * @param[in] m0 First row of the filter.
324 * @param[in] m1 Second row of the filter.
325 * @param[in] m2 Third row of the filter.
326 * @param[in] fixed_point_position (Optional) Fixed point position.
327 * @param[in] input_offset (Optional) Input quantization offset.
328 *
329 */
330template <unsigned int stridex>
331int32x4x2_t convolve_3x3(const uint8_t *in_top, const uint8_t *in_mid, const uint8_t *in_low,
332 const int32x4x3_t &m0, const int32x4x3_t &m1, const int32x4x3_t &m2,
333 int fixed_point_position, int input_offset);
334
335template <>
336inline int32x4x2_t convolve_3x3<1>(const uint8_t *in_top, const uint8_t *in_mid, const uint8_t *in_low, const int32x4x3_t &m0, const int32x4x3_t &m1, const int32x4x3_t &m2,
337 int fixed_point_position, int input_offset)
338{
339 ARM_COMPUTE_UNUSED(fixed_point_position);
340
341 const int32x4_t v_input_offset = vdupq_n_s32(input_offset);
342
343 const uint8x8x2_t vtop =
344 {
345 {
346 vld1_u8(in_top),
347 vld1_u8(in_top + 8)
348 }
349 };
350 const uint8x8x2_t vmid =
351 {
352 {
353 vld1_u8(in_mid),
354 vld1_u8(in_mid + 8)
355 }
356 };
357 const uint8x8x2_t vlow =
358 {
359 {
360 vld1_u8(in_low),
361 vld1_u8(in_low + 8)
362 }
363 };
364
365 const int32x4x3_t vtop_s32 =
366 {
367 {
368 vaddw_s16(v_input_offset, vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vtop.val[0])))),
369 vaddw_s16(v_input_offset, vreinterpret_s16_u16(vget_high_u16(vmovl_u8(vtop.val[0])))),
370 vaddw_s16(v_input_offset, vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vtop.val[1])))),
371 }
372 };
373 const int32x4x3_t vmid_s32 =
374 {
375 {
376 vaddw_s16(v_input_offset, vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vmid.val[0])))),
377 vaddw_s16(v_input_offset, vreinterpret_s16_u16(vget_high_u16(vmovl_u8(vmid.val[0])))),
378 vaddw_s16(v_input_offset, vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vmid.val[1])))),
379 }
380 };
381 const int32x4x3_t vlow_s32 =
382 {
383 {
384 vaddw_s16(v_input_offset, vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vlow.val[0])))),
385 vaddw_s16(v_input_offset, vreinterpret_s16_u16(vget_high_u16(vmovl_u8(vlow.val[0])))),
386 vaddw_s16(v_input_offset, vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vlow.val[1])))),
387 }
388 };
389
390 int32x4x2_t out
391 {
392 {
393 vdupq_n_s32(0),
394 vdupq_n_s32(0),
395 }
396 };
397
398 // 0
399 out.val[0] = vmlaq_s32(out.val[0], vtop_s32.val[0], m0.val[0]);
400 out.val[0] = vmlaq_s32(out.val[0], vextq_s32(vtop_s32.val[0], vtop_s32.val[1], 1), m0.val[1]);
401 out.val[0] = vmlaq_s32(out.val[0], vextq_s32(vtop_s32.val[0], vtop_s32.val[1], 2), m0.val[2]);
402
403 out.val[0] = vmlaq_s32(out.val[0], vmid_s32.val[0], m1.val[0]);
404 out.val[0] = vmlaq_s32(out.val[0], vextq_s32(vmid_s32.val[0], vmid_s32.val[1], 1), m1.val[1]);
405 out.val[0] = vmlaq_s32(out.val[0], vextq_s32(vmid_s32.val[0], vmid_s32.val[1], 2), m1.val[2]);
406
407 out.val[0] = vmlaq_s32(out.val[0], vlow_s32.val[0], m2.val[0]);
408 out.val[0] = vmlaq_s32(out.val[0], vextq_s32(vlow_s32.val[0], vlow_s32.val[1], 1), m2.val[1]);
409 out.val[0] = vmlaq_s32(out.val[0], vextq_s32(vlow_s32.val[0], vlow_s32.val[1], 2), m2.val[2]);
410
411 // 1
412 out.val[1] = vmlaq_s32(out.val[1], vtop_s32.val[1], m0.val[0]);
413 out.val[1] = vmlaq_s32(out.val[1], vextq_s32(vtop_s32.val[1], vtop_s32.val[2], 1), m0.val[1]);
414 out.val[1] = vmlaq_s32(out.val[1], vextq_s32(vtop_s32.val[1], vtop_s32.val[2], 2), m0.val[2]);
415
416 out.val[1] = vmlaq_s32(out.val[1], vmid_s32.val[1], m1.val[0]);
417 out.val[1] = vmlaq_s32(out.val[1], vextq_s32(vmid_s32.val[1], vmid_s32.val[2], 1), m1.val[1]);
418 out.val[1] = vmlaq_s32(out.val[1], vextq_s32(vmid_s32.val[1], vmid_s32.val[2], 2), m1.val[2]);
419
420 out.val[1] = vmlaq_s32(out.val[1], vlow_s32.val[1], m2.val[0]);
421 out.val[1] = vmlaq_s32(out.val[1], vextq_s32(vlow_s32.val[1], vlow_s32.val[2], 1), m2.val[1]);
422 out.val[1] = vmlaq_s32(out.val[1], vextq_s32(vlow_s32.val[1], vlow_s32.val[2], 2), m2.val[2]);
423
424 return out;
425}
426
427template <>
428inline int32x4x2_t convolve_3x3<2>(const uint8_t *in_top, const uint8_t *in_mid, const uint8_t *in_low,
429 const int32x4x3_t &m0, const int32x4x3_t &m1, const int32x4x3_t &m2,
430 int fixed_point_position, int input_offset)
431{
432 ARM_COMPUTE_UNUSED(fixed_point_position);
433
434 int32x4x2_t out = convolve_3x3<1>(in_top, in_mid, in_low, m0, m1, m2, fixed_point_position, input_offset);
435 out.val[0] = vsetq_lane_s32(vgetq_lane_s32(out.val[0], 2), out.val[0], 1);
436 out.val[0] = vsetq_lane_s32(vgetq_lane_s32(out.val[1], 0), out.val[0], 2);
437 out.val[0] = vsetq_lane_s32(vgetq_lane_s32(out.val[1], 2), out.val[0], 3);
438 return out;
439}
440
441template <>
442inline int32x4x2_t convolve_3x3<3>(const uint8_t *in_top, const uint8_t *in_mid, const uint8_t *in_low,
443 const int32x4x3_t &m0, const int32x4x3_t &m1, const int32x4x3_t &m2,
444 int fixed_point_position, int input_offset)
445{
446 ARM_COMPUTE_UNUSED(fixed_point_position);
447 int32x4x2_t out = convolve_3x3<1>(in_top, in_mid, in_low, m0, m1, m2, fixed_point_position, input_offset);
448 out.val[0] = vsetq_lane_s32(vgetq_lane_s32(out.val[0], 3), out.val[0], 1);
449 return out;
450}
451
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100452/** Stores a float32x4x2_t array into a memory location.
453 *
454 * @param[in] buffer Pointer to the memory location where the values will be stored.
455 * @param[in] values Values that will be stored.
456 *
457 */
458template <unsigned int stridex>
459void store_results(float *buffer, const float32x4x2_t &values);
460
461template <>
462inline void store_results<1>(float *buffer, const float32x4x2_t &values)
463{
464 vst1q_f32(buffer, values.val[0]);
465 vst1q_f32(buffer + 4, values.val[1]);
466}
467
468template <>
469inline void store_results<2>(float *buffer, const float32x4x2_t &values)
470{
471 vst1q_f32(buffer, values.val[0]);
472}
473
474template <>
475inline void store_results<3>(float *buffer, const float32x4x2_t &values)
476{
477 vst1_f32(buffer, vget_low_f32(values.val[0]));
478}
479
480/** Stores a qint16_t array into a memory location.
481 *
482 * @param[in] buffer Pointer to the memory location where the values will be stored.
483 * @param[in] values Values that will be stored.
484 *
485 */
486template <unsigned int stridex>
487void store_results(qint16_t *buffer, const qint16x8x2_t &values);
488
489template <>
490inline void store_results<1>(qint16_t *buffer, const qint16x8x2_t &values)
491{
492 vst1q_qs16(buffer, values.val[0]);
493 vst1q_qs16(buffer + 8, values.val[1]);
494}
495
496template <>
497inline void store_results<2>(qint16_t *buffer, const qint16x8x2_t &values)
498{
499 vst1q_qs16(buffer, values.val[0]);
500}
501
502template <>
503inline void store_results<3>(qint16_t *buffer, const qint16x8x2_t &values)
504{
505 vst1_qs16(buffer, vget_low_s16(values.val[0]));
506}
507
Georgios Pinitasf72f9362018-01-12 16:29:45 +0000508/** Stores a uint32_t array into a memory location.
509 *
510 * @param[in] buffer Pointer to the memory location where the values will be stored.
511 * @param[in] values Values that will be stored.
512 *
513 */
514template <unsigned int stridex>
515void store_results(int32_t *buffer, const int32x4x2_t &values);
516
517template <>
518inline void store_results<1>(int32_t *buffer, const int32x4x2_t &values)
519{
520 vst1q_s32(buffer, values.val[0]);
521 vst1q_s32(buffer + 4, values.val[1]);
522}
523
524template <>
525inline void store_results<2>(int32_t *buffer, const int32x4x2_t &values)
526{
527 vst1q_s32(buffer, values.val[0]);
528}
529
530template <>
531inline void store_results<3>(int32_t *buffer, const int32x4x2_t &values)
532{
533 vst1_s32(buffer, vget_low_s32(values.val[0]));
534}
535
Ioan-Cristian Szabo5edbd1c2017-11-13 13:34:08 +0000536#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100537/** Loads a 3x3 matrix as a row (float16_t).
538 *
539 * @param[in] ptr Pointer to a float 3x3 matrix.
540 *
541 * @return The loaded matrix.
542 */
543inline float16x8x3_t load_matrix_row(const float16_t *ptr)
544{
545 /* ptr is a pointer to a row in a 3x3 matrix, the function returns 3 vectors holding exactly the same value in all lanes:
546 r.val[0] contains the first element, r.val[1] the second element and r.val[2] the third element (in all lanes) */
547 const float16x8x3_t r =
548 {
549 {
550 vld1q_dup_f16(ptr),
551 vld1q_dup_f16(1 + ptr),
552 vld1q_dup_f16(2 + ptr)
553 }
554 };
555 return r;
556}
557
558/** Perform a convolve3x3 on float16.
559 *
560 * @param[in] in_top Pointer to the first row of the input.
561 * @param[in] in_mid Pointer to the second row of the input.
562 * @param[in] in_low Pointer to the third row of the input.
563 * @param[in] m0 First row of the filter.
564 * @param[in] m1 Second row of the filter.
565 * @param[in] m2 Third row of the filter.
566 * @param[in] fixed_point_position (Optional) Fixed point position.
567 *
568 */
569template <unsigned int stridex>
570float16x8x2_t convolve_3x3(const float16_t *in_top, const float16_t *in_mid, const float16_t *in_low, const float16x8x3_t &m0, const float16x8x3_t &m1, const float16x8x3_t &m2,
571 int fixed_point_position);
572
573template <>
574inline float16x8x2_t convolve_3x3<1>(const float16_t *in_top, const float16_t *in_mid, const float16_t *in_low, const float16x8x3_t &m0, const float16x8x3_t &m1, const float16x8x3_t &m2,
575 int fixed_point_position)
576{
577 ARM_COMPUTE_UNUSED(fixed_point_position);
578
579 const float16x8x3_t vtop =
580 {
581 {
582 vld1q_f16(in_top),
583 vld1q_f16(in_top + 8),
584 vld1q_f16(in_top + 16)
585 }
586 };
587 const float16x8x3_t vmid =
588 {
589 {
590 vld1q_f16(in_mid),
591 vld1q_f16(in_mid + 8),
592 vld1q_f16(in_mid + 16)
593 }
594 };
595 const float16x8x3_t vlow =
596 {
597 {
598 vld1q_f16(in_low),
599 vld1q_f16(in_low + 8),
600 vld1q_f16(in_low + 16)
601 }
602 };
603 float16x8x2_t out =
604 {
605 {
606 vmulq_f16(vtop.val[0], m0.val[0]),
607 vmulq_f16(vtop.val[1], m0.val[0])
608 }
609 };
610 out.val[0] = vaddq_f16(out.val[0], vmulq_f16(vextq_f16(vtop.val[0], vtop.val[1], 1), m0.val[1]));
611 out.val[0] = vaddq_f16(out.val[0], vmulq_f16(vextq_f16(vtop.val[0], vtop.val[1], 2), m0.val[2]));
612 out.val[0] = vaddq_f16(out.val[0], vmulq_f16(vmid.val[0], m1.val[0]));
613 out.val[0] = vaddq_f16(out.val[0], vmulq_f16(vextq_f16(vmid.val[0], vmid.val[1], 1), m1.val[1]));
614 out.val[0] = vaddq_f16(out.val[0], vmulq_f16(vextq_f16(vmid.val[0], vmid.val[1], 2), m1.val[2]));
615 out.val[0] = vaddq_f16(out.val[0], vmulq_f16(vlow.val[0], m2.val[0]));
616 out.val[0] = vaddq_f16(out.val[0], vmulq_f16(vextq_f16(vlow.val[0], vlow.val[1], 1), m2.val[1]));
617 out.val[0] = vaddq_f16(out.val[0], vmulq_f16(vextq_f16(vlow.val[0], vlow.val[1], 2), m2.val[2]));
618 out.val[1] = vaddq_f16(out.val[1], vmulq_f16(vextq_f16(vtop.val[1], vtop.val[2], 1), m0.val[1]));
619 out.val[1] = vaddq_f16(out.val[1], vmulq_f16(vextq_f16(vtop.val[1], vtop.val[2], 2), m0.val[2]));
620 out.val[1] = vaddq_f16(out.val[1], vmulq_f16(vmid.val[1], m1.val[0]));
621 out.val[1] = vaddq_f16(out.val[1], vmulq_f16(vextq_f16(vmid.val[1], vmid.val[2], 1), m1.val[1]));
622 out.val[1] = vaddq_f16(out.val[1], vmulq_f16(vextq_f16(vmid.val[1], vmid.val[2], 2), m1.val[2]));
623 out.val[1] = vaddq_f16(out.val[1], vmulq_f16(vlow.val[1], m2.val[0]));
624 out.val[1] = vaddq_f16(out.val[1], vmulq_f16(vextq_f16(vlow.val[1], vlow.val[2], 1), m2.val[1]));
625 out.val[1] = vaddq_f16(out.val[1], vmulq_f16(vextq_f16(vlow.val[1], vlow.val[2], 2), m2.val[2]));
626 return out;
627}
628
629template <>
630inline float16x8x2_t convolve_3x3<2>(const float16_t *in_top, const float16_t *in_mid, const float16_t *in_low, const float16x8x3_t &m0, const float16x8x3_t &m1, const float16x8x3_t &m2,
631 int fixed_point_position)
632{
633 float16x8x2_t out = convolve_3x3<1>(in_top, in_mid, in_low, m0, m1, m2, fixed_point_position);
634 out.val[0] = vsetq_lane_f16(vgetq_lane_f16(out.val[0], 2), out.val[0], 1);
635 out.val[0] = vsetq_lane_f16(vgetq_lane_f16(out.val[1], 0), out.val[0], 2);
636 out.val[0] = vsetq_lane_f16(vgetq_lane_f16(out.val[1], 2), out.val[0], 3);
637 return out;
638}
639
640template <>
641inline float16x8x2_t convolve_3x3<3>(const float16_t *in_top, const float16_t *in_mid, const float16_t *in_low, const float16x8x3_t &m0, const float16x8x3_t &m1, const float16x8x3_t &m2,
642 int fixed_point_position)
643{
644 float16x8x2_t out = convolve_3x3<1>(in_top, in_mid, in_low, m0, m1, m2, fixed_point_position);
645 out.val[0] = vsetq_lane_f16(vgetq_lane_f16(out.val[0], 3), out.val[0], 1);
646 return out;
647}
648
649/** Stores a float16x8x2_t array into a memory location.
650 *
651 * @param[in] buffer Pointer to the memory location where the values will be stored.
652 * @param[in] values Values that will be stored.
653 *
654 */
655template <unsigned int stridex>
656void store_results(float16_t *buffer, const float16x8x2_t &values);
657
658template <>
659inline void store_results<1>(float16_t *buffer, const float16x8x2_t &values)
660{
661 vst1q_f16(buffer, values.val[0]);
662 vst1q_f16(buffer + 8, values.val[1]);
663}
664
665template <>
666inline void store_results<2>(float16_t *buffer, const float16x8x2_t &values)
667{
668 vst1q_f16(buffer, values.val[0]);
669}
670
671template <>
672inline void store_results<3>(float16_t *buffer, const float16x8x2_t &values)
673{
674 vst1_f16(buffer, vget_low_f16(values.val[0]));
675}
Ioan-Cristian Szabo5edbd1c2017-11-13 13:34:08 +0000676#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100677
678/** Get the number of elements processed on 3x3 convolution.
679 *
680 * @param[in] num_elems_written_per_iteration Number of elements written per iteration on 3x3 convolution.
681 *
682 * @return The number of elements processed.
683 */
684template <unsigned int stridex>
685int get_input_num_elems_processed(unsigned int num_elems_written_per_iteration);
686
687template <>
688inline int get_input_num_elems_processed<1>(unsigned int num_elems_written_per_iteration)
689{
690 return num_elems_written_per_iteration;
691}
692
693template <>
694inline int get_input_num_elems_processed<2>(unsigned int num_elems_written_per_iteration)
695{
696 return num_elems_written_per_iteration << 1;
697}
698
699template <>
700inline int get_input_num_elems_processed<3>(unsigned int num_elems_written_per_iteration)
701{
702 return num_elems_written_per_iteration * 3;
703}
Anthony Barbier15686212017-12-12 17:17:50 +0000704inline int get_input_num_elems_processed(unsigned int num_elems_written_per_iteration, unsigned int stridex)
705{
706 switch(stridex)
707 {
708 case 1:
709 return get_input_num_elems_processed<1>(num_elems_written_per_iteration);
710 case 2:
711 return get_input_num_elems_processed<2>(num_elems_written_per_iteration);
712 case 3:
713 return get_input_num_elems_processed<3>(num_elems_written_per_iteration);
714 default:
715 ARM_COMPUTE_ERROR("stridex not supported");
716 return 0;
717 }
718}
Michalis Spyrou7362f0d2017-10-18 17:58:22 +0100719}
720} // namespace arm_compute
Anthony Barbier15686212017-12-12 17:17:50 +0000721#endif /* __ARM_COMPUTE_NEDIRECTCONVOLUTIONDETAIL_H__ */