Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / 00afd11eaa7d408ff873732639c9a724fece9058 / . / src / core / NEON / kernels / winograd / transforms / kernel_2x2_3x3 / a64_float.hpp
blob: 3dd62d1ac1bf67c946d0b8e61f61a878a0d8e725 [file] [log] [blame]
Pablo Tello89519332017-11-17 11:52:36 +0000[diff] [blame]1/*
2 * Copyright (c) 2017 ARM Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#pragma once
25
26#ifdef __aarch64__
27namespace winograd {
28template <>
29template <>
30inline void winograd2x2_3x3_gemm_kernel_transform_impl<float>::transform_kernel<0>(
31 const float* const kernel,
32 const int n_input_channels,
33 const int n_output_channels,
34 float* const matrix_base,
35 const int mstride,
36 const int matrix_row_stride
37) {
38 // Use one input pointer for each row of the kernel, use two additional
39 // offsets to extract columns.
40 const int kernel_col_stride = n_input_channels * n_output_channels;
41 const int kernel_row_stride = 3 * kernel_col_stride;
42 const float *inptr0 = kernel;
43 const float *inptr1 = kernel + kernel_row_stride;
44 const float *inptr2 = kernel + kernel_row_stride*2;
45
46 // Use four output pointers, for output matrices 0, 4, 8 and 12. Use three
47 // offsets to extract further matrices.
48 float *outptr0 = matrix_base;
49 float *outptr4 = matrix_base + mstride * 4;
50 float *outptr8 = matrix_base + mstride * 8;
51 float *outptr12 = matrix_base + mstride * 12;
52
53 // For every input channel
54 for (int in_c = 0; in_c < n_input_channels; in_c++) {
55 int n_remaining_channels = n_output_channels;
56
57 asm volatile (
58 // Registers into which to read the kernel
59 "w_11 .req v0\n" "qw_11 .req q0\n"
60 "w_12 .req v1\n" "qw_12 .req q1\n"
61 "w_13 .req v2\n" "qw_13 .req q2\n"
62 "w_21 .req v3\n" "qw_21 .req q3\n"
63 "w_22 .req v4\n" "qw_22 .req q4\n"
64 "w_23 .req v5\n" "qw_23 .req q5\n"
65 "w_31 .req v6\n" "qw_31 .req q6\n"
66 "w_32 .req v7\n" "qw_32 .req q7\n"
67 "w_33 .req v8\n" "qw_33 .req q8\n"
68
69 // Transformed matrix Ww
70 "Ww11 .req w_11\n" "Ww12 .req w_12\n" "Ww13 .req w_13\n"
71 "Ww21 .req v9\n" "Ww22 .req v10\n" "Ww23 .req v11\n"
72 "Ww31 .req v12\n" "Ww32 .req v13\n" "Ww33 .req v14\n"
73 "Ww41 .req w_31\n" "Ww42 .req w_32\n" "Ww43 .req w_33\n"
74
75 // Output matrix U = WwWT
76 "U11 .req Ww11\n" "U12 .req v15\n" "U13 .req v16\n" "U14 .req Ww13\n"
77 "U21 .req Ww21\n" "U22 .req v17\n" "U23 .req v18\n" "U24 .req Ww23\n"
78 "U31 .req Ww31\n" "U32 .req v19\n" "U33 .req v20\n" "U34 .req Ww33\n"
79 "U41 .req Ww41\n" "U42 .req v21\n" "U43 .req v22\n" "U44 .req Ww43\n"
80
81 // Storage view of output matrices
82 "qU11 .req q0\n" "qU12 .req q15\n" "qU13 .req q16\n" "qU14 .req q2\n"
83 "qU21 .req q9\n" "qU22 .req q17\n" "qU23 .req q18\n" "qU24 .req q11\n"
84 "qU31 .req q12\n" "qU32 .req q19\n" "qU33 .req q20\n" "qU34 .req q14\n"
85 "qU41 .req q6\n" "qU42 .req q21\n" "qU43 .req q22\n" "qU44 .req q8\n"
86
87 "half .req v23\n" // {0.5, ..., 0.5}
88 "dup half.4s, %w[one_half]\n"
89 "scratch .req v24\n"
90
91 "1:"
92 // Load tile of the kernel
93 "ldr qw_11, [%x[inptr0]]\n"
94 "str qU11, [%x[outptr0]]\n"
95 "ldr qw_12, [%x[inptr0], %x[colstride1]]\n"
96 "ldr qw_13, [%x[inptr0], %x[colstride2]]\n"
97 "str qU14, [%x[outptr0], %x[mstride3]]\n"
98 "add %x[inptr0], %x[inptr0], #0x10\n"
99
100 "ldr qw_21, [%x[inptr1]]\n"
101 "ldr qw_22, [%x[inptr1], %x[colstride1]]\n"
102 "ldr qw_23, [%x[inptr1], %x[colstride2]]\n"
103 "add %x[inptr1], %x[inptr1], #0x10\n"
104
105 "ldr qw_31, [%x[inptr2]]\n"
106 "str qU41, [%x[outptr12]]\n"
107 "ldr qw_32, [%x[inptr2], %x[colstride1]]\n"
108 "ldr qw_33, [%x[inptr2], %x[colstride2]]\n"
109 "str qU44, [%x[outptr12], %x[mstride3]]\n"
110 "add %x[inptr2], %x[inptr2], #0x10\n"
111
112 // Compute 2nd and 3rd rows of Ww
113 "fadd scratch.4s, w_11.4s, w_31.4s\n"
114 "fmul Ww21.4s, scratch.4s, half.4s\n"
115 "fmla Ww21.4s, w_21.4s, half.4s\n"
116 "str qU21, [%x[outptr4]]\n"
117 "fmul Ww31.4s, scratch.4s, half.4s\n"
118 "fmls Ww31.4s, w_21.4s, half.4s\n"
119 "str qU31, [%x[outptr8]]\n"
120
121 "fadd scratch.4s, w_12.4s, w_32.4s\n"
122 "fmul Ww22.4s, scratch.4s, half.4s\n"
123 "fmla Ww22.4s, w_22.4s, half.4s\n"
124 "fmul Ww32.4s, scratch.4s, half.4s\n"
125 "fmls Ww32.4s, w_22.4s, half.4s\n"
126
127 "fadd scratch.4s, w_13.4s, w_33.4s\n"
128 "fmul Ww23.4s, scratch.4s, half.4s\n"
129 "fmla Ww23.4s, w_23.4s, half.4s\n"
130 "str qU24, [%x[outptr4], %x[mstride3]]\n"
131 "fmul Ww33.4s, scratch.4s, half.4s\n"
132 "fmls Ww33.4s, w_23.4s, half.4s\n"
133 "str qU34, [%x[outptr8], %x[mstride3]]\n"
134
135 // Compute and store U, only need to compute the 2nd and 3rd columns
136 // of U and update output pointers
137 "fadd scratch.4s, Ww11.4s, Ww13.4s\n"
138 "fmul U12.4s, scratch.4s, half.4s\n"
139 "fmla U12.4s, Ww12.4s, half.4s\n"
140 "str qU12, [%x[outptr0], %x[mstride1]]\n"
141 "fmul U13.4s, scratch.4s, half.4s\n"
142 "fmls U13.4s, Ww12.4s, half.4s\n"
143 "str qU13, [%x[outptr0], %x[mstride2]]\n"
144 "add %x[outptr0], %x[outptr0], #0x10\n"
145
146 "fadd scratch.4s, Ww21.4s, Ww23.4s\n"
147 "fmul U22.4s, scratch.4s, half.4s\n"
148 "fmla U22.4s, Ww22.4s, half.4s\n"
149 "str qU22, [%x[outptr4], %x[mstride1]]\n"
150 "fmul U23.4s, scratch.4s, half.4s\n"
151 "fmls U23.4s, Ww22.4s, half.4s\n"
152 "str qU23, [%x[outptr4], %x[mstride2]]\n"
153 "add %x[outptr4], %x[outptr4], #0x10\n"
154
155 "fadd scratch.4s, Ww31.4s, Ww33.4s\n"
156 "fmul U32.4s, scratch.4s, half.4s\n"
157 "fmla U32.4s, Ww32.4s, half.4s\n"
158 "str qU32, [%x[outptr8], %x[mstride1]]\n"
159 "fmul U33.4s, scratch.4s, half.4s\n"
160 "fmls U33.4s, Ww32.4s, half.4s\n"
161 "str qU33, [%x[outptr8], %x[mstride2]]\n"
162 "add %x[outptr8], %x[outptr8], #0x10\n"
163
164 "fadd scratch.4s, Ww41.4s, Ww43.4s\n"
165 "fmul U42.4s, scratch.4s, half.4s\n"
166 "fmla U42.4s, Ww42.4s, half.4s\n"
167 "str qU42, [%x[outptr12], %x[mstride1]]\n"
168 "fmul U43.4s, scratch.4s, half.4s\n"
169 "fmls U43.4s, Ww42.4s, half.4s\n"
170 "str qU43, [%x[outptr12], %x[mstride2]]\n"
171 "add %x[outptr12], %x[outptr12], #0x10\n"
172
173 "subs %x[n_remaining_channels], %x[n_remaining_channels], #4\n"
174 "bne 1b\n"
175
176 // Clear aliases
177 ".unreq half\n"
178 ".unreq scratch\n"
179 ".unreq w_11\n" ".unreq qw_11\n"
180 ".unreq w_12\n" ".unreq qw_12\n"
181 ".unreq w_13\n" ".unreq qw_13\n"
182 ".unreq w_21\n" ".unreq qw_21\n"
183 ".unreq w_22\n" ".unreq qw_22\n"
184 ".unreq w_23\n" ".unreq qw_23\n"
185 ".unreq w_31\n" ".unreq qw_31\n"
186 ".unreq w_32\n" ".unreq qw_32\n"
187 ".unreq w_33\n" ".unreq qw_33\n"
188 ".unreq Ww11\n" ".unreq Ww12\n" ".unreq Ww13\n"
189 ".unreq Ww21\n" ".unreq Ww22\n" ".unreq Ww23\n"
190 ".unreq Ww31\n" ".unreq Ww32\n" ".unreq Ww33\n"
191 ".unreq Ww41\n" ".unreq Ww42\n" ".unreq Ww43\n"
192 ".unreq U11\n" ".unreq U12\n" ".unreq U13\n" ".unreq U14\n"
193 ".unreq U21\n" ".unreq U22\n" ".unreq U23\n" ".unreq U24\n"
194 ".unreq U31\n" ".unreq U32\n" ".unreq U33\n" ".unreq U34\n"
195 ".unreq U41\n" ".unreq U42\n" ".unreq U43\n" ".unreq U44\n"
196 ".unreq qU11\n" ".unreq qU12\n" ".unreq qU13\n" ".unreq qU14\n"
197 ".unreq qU21\n" ".unreq qU22\n" ".unreq qU23\n" ".unreq qU24\n"
198 ".unreq qU31\n" ".unreq qU32\n" ".unreq qU33\n" ".unreq qU34\n"
199 ".unreq qU41\n" ".unreq qU42\n" ".unreq qU43\n" ".unreq qU44\n"
200
201 : [inptr0] "+r" (inptr0),
202 [inptr1] "+r" (inptr1),
203 [inptr2] "+r" (inptr2),
204 [outptr0] "+r" (outptr0),
205 [outptr4] "+r" (outptr4),
206 [outptr8] "+r" (outptr8),
207 [outptr12] "+r" (outptr12),
208 [n_remaining_channels] "+r" (n_remaining_channels)
209 : [mstride1] "r" (sizeof(float) * mstride),
210 [mstride2] "r" (sizeof(float) * mstride * 2),
211 [mstride3] "r" (sizeof(float) * mstride * 3),
212 [colstride1] "r" (sizeof(float) * kernel_col_stride),
213 [colstride2] "r" (sizeof(float) * kernel_col_stride * 2),
214 [one_half] "r" (0.5f)
215 : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
216 "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
217 "v20", "v21", "v22", "v23", "v24"
218 );
219
220 // Progression to complete stride
221 outptr0 += matrix_row_stride - n_output_channels;
222 outptr4 += matrix_row_stride - n_output_channels;
223 outptr8 += matrix_row_stride - n_output_channels;
224 outptr12 += matrix_row_stride - n_output_channels;
225 }
226}
227
228template <>
229template <>
230inline void winograd2x2_3x3_gemm_kernel_transform_impl<float>::transform_kernel<2>(
231 const float* const kernel,
232 const int n_input_channels,
233 const int n_output_channels,
234 float* const matrix_base,
235 const int mstride,
236 const int matrix_row_stride
237) {
238 // Use one input pointer for each row of the kernel, use two additional
239 // offsets to extract columns.
240 const int kernel_col_stride = n_input_channels * n_output_channels;
241 const int kernel_row_stride = 3 * kernel_col_stride;
242 const float *inptr0 = kernel;
243 const float *inptr1 = kernel + kernel_row_stride;
244 const float *inptr2 = kernel + kernel_row_stride*2;
245
246 // Use four output pointers, for output matrices 0, 4, 8 and 12. Use three
247 // offsets to extract further matrices.
248 float *outptr0 = matrix_base;
249 float *outptr4 = matrix_base + mstride * 4;
250 float *outptr8 = matrix_base + mstride * 8;
251 float *outptr12 = matrix_base + mstride * 12;
252
253 // For every input channel
254 for (int in_c = 0; in_c < n_input_channels; in_c++) {
255 int n_remaining_channels = n_output_channels;
256
257 asm volatile (
258 // Registers into which to read the kernel
259 "w_11 .req v0\n" "qw_11 .req q0\n" "dw_11 .req d0\n"
260 "w_12 .req v1\n" "qw_12 .req q1\n" "dw_12 .req d1\n"
261 "w_13 .req v2\n" "qw_13 .req q2\n" "dw_13 .req d2\n"
262 "w_21 .req v3\n" "qw_21 .req q3\n" "dw_21 .req d3\n"
263 "w_22 .req v4\n" "qw_22 .req q4\n" "dw_22 .req d4\n"
264 "w_23 .req v5\n" "qw_23 .req q5\n" "dw_23 .req d5\n"
265 "w_31 .req v6\n" "qw_31 .req q6\n" "dw_31 .req d6\n"
266 "w_32 .req v7\n" "qw_32 .req q7\n" "dw_32 .req d7\n"
267 "w_33 .req v8\n" "qw_33 .req q8\n" "dw_33 .req d8\n"
268
269 // Transformed matrix Ww
270 "Ww11 .req w_11\n" "Ww12 .req w_12\n" "Ww13 .req w_13\n"
271 "Ww21 .req v9\n" "Ww22 .req v10\n" "Ww23 .req v11\n"
272 "Ww31 .req v12\n" "Ww32 .req v13\n" "Ww33 .req v14\n"
273 "Ww41 .req w_31\n" "Ww42 .req w_32\n" "Ww43 .req w_33\n"
274
275 // Output matrix U = WwWT
276 "U11 .req Ww11\n" "U12 .req v15\n" "U13 .req v16\n" "U14 .req Ww13\n"
277 "U21 .req Ww21\n" "U22 .req v17\n" "U23 .req v18\n" "U24 .req Ww23\n"
278 "U31 .req Ww31\n" "U32 .req v19\n" "U33 .req v20\n" "U34 .req Ww33\n"
279 "U41 .req Ww41\n" "U42 .req v21\n" "U43 .req v22\n" "U44 .req Ww43\n"
280
281 // Storage view of output matrices
282 "qU11 .req q0\n" "qU12 .req q15\n" "qU13 .req q16\n" "qU14 .req q2\n"
283 "qU21 .req q9\n" "qU22 .req q17\n" "qU23 .req q18\n" "qU24 .req q11\n"
284 "qU31 .req q12\n" "qU32 .req q19\n" "qU33 .req q20\n" "qU34 .req q14\n"
285 "qU41 .req q6\n" "qU42 .req q21\n" "qU43 .req q22\n" "qU44 .req q8\n"
286
287 "dU11 .req d0\n" "dU12 .req d15\n" "dU13 .req d16\n" "dU14 .req d2\n"
288 "dU21 .req d9\n" "dU22 .req d17\n" "dU23 .req d18\n" "dU24 .req d11\n"
289 "dU31 .req d12\n" "dU32 .req d19\n" "dU33 .req d20\n" "dU34 .req d14\n"
290 "dU41 .req d6\n" "dU42 .req d21\n" "dU43 .req d22\n" "dU44 .req d8\n"
291
292 "half .req v23\n" // {0.5, ..., 0.5}
293 "dup half.4s, %w[one_half]\n"
294 "scratch .req v24\n"
295
296 // Subtract the tail from the number of remaining channels and jump to
297 // the tail if necessary.
298 "subs %x[n_remaining_channels], %x[n_remaining_channels], #2\n"
299 "beq 2f\n"
300
301 "1:"
302 // Load tile of the kernel
303 "ldr qw_11, [%x[inptr0]]\n"
304 "str qU11, [%x[outptr0]]\n"
305 "ldr qw_12, [%x[inptr0], %x[colstride1]]\n"
306 "ldr qw_13, [%x[inptr0], %x[colstride2]]\n"
307 "str qU14, [%x[outptr0], %x[mstride3]]\n"
308 "add %x[inptr0], %x[inptr0], #0x10\n"
309
310 "ldr qw_21, [%x[inptr1]]\n"
311 "ldr qw_22, [%x[inptr1], %x[colstride1]]\n"
312 "ldr qw_23, [%x[inptr1], %x[colstride2]]\n"
313 "add %x[inptr1], %x[inptr1], #0x10\n"
314
315 "ldr qw_31, [%x[inptr2]]\n"
316 "str qU41, [%x[outptr12]]\n"
317 "ldr qw_32, [%x[inptr2], %x[colstride1]]\n"
318 "ldr qw_33, [%x[inptr2], %x[colstride2]]\n"
319 "str qU44, [%x[outptr12], %x[mstride3]]\n"
320 "add %x[inptr2], %x[inptr2], #0x10\n"
321
322 // Compute 2nd and 3rd rows of Ww
323 "fadd scratch.4s, w_11.4s, w_31.4s\n"
324 "fmul Ww21.4s, scratch.4s, half.4s\n"
325 "fmla Ww21.4s, w_21.4s, half.4s\n"
326 "str qU21, [%x[outptr4]]\n"
327 "fmul Ww31.4s, scratch.4s, half.4s\n"
328 "fmls Ww31.4s, w_21.4s, half.4s\n"
329 "str qU31, [%x[outptr8]]\n"
330
331 "fadd scratch.4s, w_12.4s, w_32.4s\n"
332 "fmul Ww22.4s, scratch.4s, half.4s\n"
333 "fmla Ww22.4s, w_22.4s, half.4s\n"
334 "fmul Ww32.4s, scratch.4s, half.4s\n"
335 "fmls Ww32.4s, w_22.4s, half.4s\n"
336
337 "fadd scratch.4s, w_13.4s, w_33.4s\n"
338 "fmul Ww23.4s, scratch.4s, half.4s\n"
339 "fmla Ww23.4s, w_23.4s, half.4s\n"
340 "str qU24, [%x[outptr4], %x[mstride3]]\n"
341 "fmul Ww33.4s, scratch.4s, half.4s\n"
342 "fmls Ww33.4s, w_23.4s, half.4s\n"
343 "str qU34, [%x[outptr8], %x[mstride3]]\n"
344
345 // Compute and store U, only need to compute the 2nd and 3rd columns
346 // of U and update output pointers
347 "fadd scratch.4s, Ww11.4s, Ww13.4s\n"
348 "fmul U12.4s, scratch.4s, half.4s\n"
349 "fmla U12.4s, Ww12.4s, half.4s\n"
350 "str qU12, [%x[outptr0], %x[mstride1]]\n"
351 "fmul U13.4s, scratch.4s, half.4s\n"
352 "fmls U13.4s, Ww12.4s, half.4s\n"
353 "str qU13, [%x[outptr0], %x[mstride2]]\n"
354 "add %x[outptr0], %x[outptr0], #0x10\n"
355
356 "fadd scratch.4s, Ww21.4s, Ww23.4s\n"
357 "fmul U22.4s, scratch.4s, half.4s\n"
358 "fmla U22.4s, Ww22.4s, half.4s\n"
359 "str qU22, [%x[outptr4], %x[mstride1]]\n"
360 "fmul U23.4s, scratch.4s, half.4s\n"
361 "fmls U23.4s, Ww22.4s, half.4s\n"
362 "str qU23, [%x[outptr4], %x[mstride2]]\n"
363 "add %x[outptr4], %x[outptr4], #0x10\n"
364
365 "fadd scratch.4s, Ww31.4s, Ww33.4s\n"
366 "fmul U32.4s, scratch.4s, half.4s\n"
367 "fmla U32.4s, Ww32.4s, half.4s\n"
368 "str qU32, [%x[outptr8], %x[mstride1]]\n"
369 "fmul U33.4s, scratch.4s, half.4s\n"
370 "fmls U33.4s, Ww32.4s, half.4s\n"
371 "str qU33, [%x[outptr8], %x[mstride2]]\n"
372 "add %x[outptr8], %x[outptr8], #0x10\n"
373
374 "fadd scratch.4s, Ww41.4s, Ww43.4s\n"
375 "fmul U42.4s, scratch.4s, half.4s\n"
376 "fmla U42.4s, Ww42.4s, half.4s\n"
377 "str qU42, [%x[outptr12], %x[mstride1]]\n"
378 "fmul U43.4s, scratch.4s, half.4s\n"
379 "fmls U43.4s, Ww42.4s, half.4s\n"
380 "str qU43, [%x[outptr12], %x[mstride2]]\n"
381 "add %x[outptr12], %x[outptr12], #0x10\n"
382
383 "subs %x[n_remaining_channels], %x[n_remaining_channels], #4\n"
384 "bne 1b\n"
385
386 // Tail size 2
387 "2:"
388 // Load tile of the kernel
389 "ldr dw_11, [%x[inptr0]]\n"
390 "str dU11, [%x[outptr0]]\n"
391 "ldr dw_12, [%x[inptr0], %x[colstride1]]\n"
392 "ldr dw_13, [%x[inptr0], %x[colstride2]]\n"
393 "str dU14, [%x[outptr0], %x[mstride3]]\n"
394 "add %x[inptr0], %x[inptr0], #0x08\n"
395
396 "ldr dw_21, [%x[inptr1]]\n"
397 "ldr dw_22, [%x[inptr1], %x[colstride1]]\n"
398 "ldr dw_23, [%x[inptr1], %x[colstride2]]\n"
399 "add %x[inptr1], %x[inptr1], #0x08\n"
400
401 "ldr dw_31, [%x[inptr2]]\n"
402 "str dU41, [%x[outptr12]]\n"
403 "ldr dw_32, [%x[inptr2], %x[colstride1]]\n"
404 "ldr dw_33, [%x[inptr2], %x[colstride2]]\n"
405 "str dU44, [%x[outptr12], %x[mstride3]]\n"
406 "add %x[inptr2], %x[inptr2], #0x08\n"
407
408 // Compute 2nd and 3rd rows of Ww
409 "fadd scratch.2s, w_11.2s, w_31.2s\n"
410 "fmul Ww21.2s, scratch.2s, half.2s\n"
411 "fmla Ww21.2s, w_21.2s, half.2s\n"
412 "str dU21, [%x[outptr4]]\n"
413 "fmul Ww31.2s, scratch.2s, half.2s\n"
414 "fmls Ww31.2s, w_21.2s, half.2s\n"
415 "str dU31, [%x[outptr8]]\n"
416
417 "fadd scratch.2s, w_12.2s, w_32.2s\n"
418 "fmul Ww22.2s, scratch.2s, half.2s\n"
419 "fmla Ww22.2s, w_22.2s, half.2s\n"
420 "fmul Ww32.2s, scratch.2s, half.2s\n"
421 "fmls Ww32.2s, w_22.2s, half.2s\n"
422
423 "fadd scratch.2s, w_13.2s, w_33.2s\n"
424 "fmul Ww23.2s, scratch.2s, half.2s\n"
425 "fmla Ww23.2s, w_23.2s, half.2s\n"
426 "str dU24, [%x[outptr4], %x[mstride3]]\n"
427 "fmul Ww33.2s, scratch.2s, half.2s\n"
428 "fmls Ww33.2s, w_23.2s, half.2s\n"
429 "str dU34, [%x[outptr8], %x[mstride3]]\n"
430
431 // Compute and store U, only need to compute the 2nd and 3rd columns of
432 // U and update output pointers
433 "fadd scratch.2s, Ww11.2s, Ww13.2s\n"
434 "fmul U12.2s, scratch.2s, half.2s\n"
435 "fmla U12.2s, Ww12.2s, half.2s\n"
436 "str dU12, [%x[outptr0], %x[mstride1]]\n"
437 "fmul U13.2s, scratch.2s, half.2s\n"
438 "fmls U13.2s, Ww12.2s, half.2s\n"
439 "str dU13, [%x[outptr0], %x[mstride2]]\n"
440 "add %x[outptr0], %x[outptr0], #0x08\n"
441
442 "fadd scratch.2s, Ww21.2s, Ww23.2s\n"
443 "fmul U22.2s, scratch.2s, half.2s\n"
444 "fmla U22.2s, Ww22.2s, half.2s\n"
445 "str dU22, [%x[outptr4], %x[mstride1]]\n"
446 "fmul U23.2s, scratch.2s, half.2s\n"
447 "fmls U23.2s, Ww22.2s, half.2s\n"
448 "str dU23, [%x[outptr4], %x[mstride2]]\n"
449 "add %x[outptr4], %x[outptr4], #0x08\n"
450
451 "fadd scratch.2s, Ww31.2s, Ww33.2s\n"
452 "fmul U32.2s, scratch.2s, half.2s\n"
453 "fmla U32.2s, Ww32.2s, half.2s\n"
454 "str dU32, [%x[outptr8], %x[mstride1]]\n"
455 "fmul U33.2s, scratch.2s, half.2s\n"
456 "fmls U33.2s, Ww32.2s, half.2s\n"
457 "str dU33, [%x[outptr8], %x[mstride2]]\n"
458 "add %x[outptr8], %x[outptr8], #0x08\n"
459
460 "fadd scratch.2s, Ww41.2s, Ww43.2s\n"
461 "fmul U42.2s, scratch.2s, half.2s\n"
462 "fmla U42.2s, Ww42.2s, half.2s\n"
463 "str dU42, [%x[outptr12], %x[mstride1]]\n"
464 "fmul U43.2s, scratch.2s, half.2s\n"
465 "fmls U43.2s, Ww42.2s, half.2s\n"
466 "str dU43, [%x[outptr12], %x[mstride2]]\n"
467 "add %x[outptr12], %x[outptr12], #0x08\n"
468
469 // Clear aliases
470 ".unreq half\n"
471 ".unreq scratch\n"
472 ".unreq w_11\n" ".unreq qw_11\n" ".unreq dw_11\n"
473 ".unreq w_12\n" ".unreq qw_12\n" ".unreq dw_12\n"
474 ".unreq w_13\n" ".unreq qw_13\n" ".unreq dw_13\n"
475 ".unreq w_21\n" ".unreq qw_21\n" ".unreq dw_21\n"
476 ".unreq w_22\n" ".unreq qw_22\n" ".unreq dw_22\n"
477 ".unreq w_23\n" ".unreq qw_23\n" ".unreq dw_23\n"
478 ".unreq w_31\n" ".unreq qw_31\n" ".unreq dw_31\n"
479 ".unreq w_32\n" ".unreq qw_32\n" ".unreq dw_32\n"
480 ".unreq w_33\n" ".unreq qw_33\n" ".unreq dw_33\n"
481 ".unreq Ww11\n" ".unreq Ww12\n" ".unreq Ww13\n"
482 ".unreq Ww21\n" ".unreq Ww22\n" ".unreq Ww23\n"
483 ".unreq Ww31\n" ".unreq Ww32\n" ".unreq Ww33\n"
484 ".unreq Ww41\n" ".unreq Ww42\n" ".unreq Ww43\n"
485 ".unreq U11\n" ".unreq U12\n" ".unreq U13\n" ".unreq U14\n"
486 ".unreq U21\n" ".unreq U22\n" ".unreq U23\n" ".unreq U24\n"
487 ".unreq U31\n" ".unreq U32\n" ".unreq U33\n" ".unreq U34\n"
488 ".unreq U41\n" ".unreq U42\n" ".unreq U43\n" ".unreq U44\n"
489 ".unreq qU11\n" ".unreq qU12\n" ".unreq qU13\n" ".unreq qU14\n"
490 ".unreq qU21\n" ".unreq qU22\n" ".unreq qU23\n" ".unreq qU24\n"
491 ".unreq qU31\n" ".unreq qU32\n" ".unreq qU33\n" ".unreq qU34\n"
492 ".unreq qU41\n" ".unreq qU42\n" ".unreq qU43\n" ".unreq qU44\n"
493 ".unreq dU11\n" ".unreq dU12\n" ".unreq dU13\n" ".unreq dU14\n"
494 ".unreq dU21\n" ".unreq dU22\n" ".unreq dU23\n" ".unreq dU24\n"
495 ".unreq dU31\n" ".unreq dU32\n" ".unreq dU33\n" ".unreq dU34\n"
496 ".unreq dU41\n" ".unreq dU42\n" ".unreq dU43\n" ".unreq dU44\n"
497
498 : [inptr0] "+r" (inptr0),
499 [inptr1] "+r" (inptr1),
500 [inptr2] "+r" (inptr2),
501 [outptr0] "+r" (outptr0),
502 [outptr4] "+r" (outptr4),
503 [outptr8] "+r" (outptr8),
504 [outptr12] "+r" (outptr12),
505 [n_remaining_channels] "+r" (n_remaining_channels)
506 : [mstride1] "r" (sizeof(float) * mstride),
507 [mstride2] "r" (sizeof(float) * mstride * 2),
508 [mstride3] "r" (sizeof(float) * mstride * 3),
509 [colstride1] "r" (sizeof(float) * kernel_col_stride),
510 [colstride2] "r" (sizeof(float) * kernel_col_stride * 2),
511 [one_half] "r" (0.5f)
512 : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
513 "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
514 "v20", "v21", "v22", "v23", "v24"
515 );
516
517 // Progression to complete stride
518 outptr0 += matrix_row_stride - n_output_channels;
519 outptr4 += matrix_row_stride - n_output_channels;
520 outptr8 += matrix_row_stride - n_output_channels;
521 outptr12 += matrix_row_stride - n_output_channels;
522 }
523}
524
525template <>
526template <>
527inline void winograd2x2_3x3_gemm_kernel_transform_impl<float>::transform_kernel<1>(
528 const float* const kernel,
529 const int n_input_channels,
530 const int n_output_channels,
531 float* const matrix_base,
532 const int mstride,
533 const int matrix_row_stride
534) {
535 // Use one input pointer for each row of the kernel, use two additional
536 // offsets to extract columns.
537 const int kernel_col_stride = n_input_channels * n_output_channels;
538 const int kernel_row_stride = 3 * kernel_col_stride;
539 const float *inptr0 = kernel;
540 const float *inptr1 = kernel + kernel_row_stride;
541 const float *inptr2 = kernel + kernel_row_stride*2;
542
543 // Use four output pointers, for output matrices 0, 4, 8 and 12. Use three
544 // offsets to extract further matrices.
545 float *outptr0 = matrix_base;
546 float *outptr4 = matrix_base + mstride * 4;
547 float *outptr8 = matrix_base + mstride * 8;
548 float *outptr12 = matrix_base + mstride * 12;
549
550 // For every input channel
551 for (int in_c = 0; in_c < n_input_channels; in_c++) {
552 int n_remaining_channels = n_output_channels;
553
554 asm volatile (
555 // Registers into which to read the kernel
556 "w_11 .req v0\n" "qw_11 .req q0\n" "sw_11 .req s0\n"
557 "w_12 .req v1\n" "qw_12 .req q1\n" "sw_12 .req s1\n"
558 "w_13 .req v2\n" "qw_13 .req q2\n" "sw_13 .req s2\n"
559 "w_21 .req v3\n" "qw_21 .req q3\n" "sw_21 .req s3\n"
560 "w_22 .req v4\n" "qw_22 .req q4\n" "sw_22 .req s4\n"
561 "w_23 .req v5\n" "qw_23 .req q5\n" "sw_23 .req s5\n"
562 "w_31 .req v6\n" "qw_31 .req q6\n" "sw_31 .req s6\n"
563 "w_32 .req v7\n" "qw_32 .req q7\n" "sw_32 .req s7\n"
564 "w_33 .req v8\n" "qw_33 .req q8\n" "sw_33 .req s8\n"
565
566 // Transformed matrix Ww
567 "Ww11 .req w_11\n" "Ww12 .req w_12\n" "Ww13 .req w_13\n"
568 "Ww21 .req v9\n" "Ww22 .req v10\n" "Ww23 .req v11\n"
569 "Ww31 .req v12\n" "Ww32 .req v13\n" "Ww33 .req v14\n"
570 "Ww41 .req w_31\n" "Ww42 .req w_32\n" "Ww43 .req w_33\n"
571
572 // Output matrix U = WwWT
573 "U11 .req Ww11\n" "U12 .req v15\n" "U13 .req v16\n" "U14 .req Ww13\n"
574 "U21 .req Ww21\n" "U22 .req v17\n" "U23 .req v18\n" "U24 .req Ww23\n"
575 "U31 .req Ww31\n" "U32 .req v19\n" "U33 .req v20\n" "U34 .req Ww33\n"
576 "U41 .req Ww41\n" "U42 .req v21\n" "U43 .req v22\n" "U44 .req Ww43\n"
577
578 // Storage view of output matrices
579 "qU11 .req q0\n" "qU12 .req q15\n" "qU13 .req q16\n" "qU14 .req q2\n"
580 "qU21 .req q9\n" "qU22 .req q17\n" "qU23 .req q18\n" "qU24 .req q11\n"
581 "qU31 .req q12\n" "qU32 .req q19\n" "qU33 .req q20\n" "qU34 .req q14\n"
582 "qU41 .req q6\n" "qU42 .req q21\n" "qU43 .req q22\n" "qU44 .req q8\n"
583
584 "sU11 .req s0\n" "sU12 .req s15\n" "sU13 .req s16\n" "sU14 .req s2\n"
585 "sU21 .req s9\n" "sU22 .req s17\n" "sU23 .req s18\n" "sU24 .req s11\n"
586 "sU31 .req s12\n" "sU32 .req s19\n" "sU33 .req s20\n" "sU34 .req s14\n"
587 "sU41 .req s6\n" "sU42 .req s21\n" "sU43 .req s22\n" "sU44 .req s8\n"
588
589 "half .req v23\n" // {0.5, ..., 0.5}
590 "dup half.4s, %w[one_half]\n"
591 "scratch .req v24\n"
592
593 // Subtract the tail from the number of remaining channels and jump to
594 // the tail if necessary.
595 "subs %x[n_remaining_channels], %x[n_remaining_channels], #1\n"
596 "beq 2f\n"
597
598 "1:"
599 // Load tile of the kernel
600 "ldr qw_11, [%x[inptr0]]\n"
601 "str qU11, [%x[outptr0]]\n"
602 "ldr qw_12, [%x[inptr0], %x[colstride1]]\n"
603 "ldr qw_13, [%x[inptr0], %x[colstride2]]\n"
604 "str qU14, [%x[outptr0], %x[mstride3]]\n"
605 "add %x[inptr0], %x[inptr0], #0x10\n"
606
607 "ldr qw_21, [%x[inptr1]]\n"
608 "ldr qw_22, [%x[inptr1], %x[colstride1]]\n"
609 "ldr qw_23, [%x[inptr1], %x[colstride2]]\n"
610 "add %x[inptr1], %x[inptr1], #0x10\n"
611
612 "ldr qw_31, [%x[inptr2]]\n"
613 "str qU41, [%x[outptr12]]\n"
614 "ldr qw_32, [%x[inptr2], %x[colstride1]]\n"
615 "ldr qw_33, [%x[inptr2], %x[colstride2]]\n"
616 "str qU44, [%x[outptr12], %x[mstride3]]\n"
617 "add %x[inptr2], %x[inptr2], #0x10\n"
618
619 // Compute 2nd and 3rd rows of Ww
620 "fadd scratch.4s, w_11.4s, w_31.4s\n"
621 "fmul Ww21.4s, scratch.4s, half.4s\n"
622 "fmla Ww21.4s, w_21.4s, half.4s\n"
623 "str qU21, [%x[outptr4]]\n"
624 "fmul Ww31.4s, scratch.4s, half.4s\n"
625 "fmls Ww31.4s, w_21.4s, half.4s\n"
626 "str qU31, [%x[outptr8]]\n"
627
628 "fadd scratch.4s, w_12.4s, w_32.4s\n"
629 "fmul Ww22.4s, scratch.4s, half.4s\n"
630 "fmla Ww22.4s, w_22.4s, half.4s\n"
631 "fmul Ww32.4s, scratch.4s, half.4s\n"
632 "fmls Ww32.4s, w_22.4s, half.4s\n"
633
634 "fadd scratch.4s, w_13.4s, w_33.4s\n"
635 "fmul Ww23.4s, scratch.4s, half.4s\n"
636 "fmla Ww23.4s, w_23.4s, half.4s\n"
637 "str qU24, [%x[outptr4], %x[mstride3]]\n"
638 "fmul Ww33.4s, scratch.4s, half.4s\n"
639 "fmls Ww33.4s, w_23.4s, half.4s\n"
640 "str qU34, [%x[outptr8], %x[mstride3]]\n"
641
642 // Compute and store U, only need to compute the 2nd and 3rd columns
643 // of U and update output pointers
644 "fadd scratch.4s, Ww11.4s, Ww13.4s\n"
645 "fmul U12.4s, scratch.4s, half.4s\n"
646 "fmla U12.4s, Ww12.4s, half.4s\n"
647 "str qU12, [%x[outptr0], %x[mstride1]]\n"
648 "fmul U13.4s, scratch.4s, half.4s\n"
649 "fmls U13.4s, Ww12.4s, half.4s\n"
650 "str qU13, [%x[outptr0], %x[mstride2]]\n"
651 "add %x[outptr0], %x[outptr0], #0x10\n"
652
653 "fadd scratch.4s, Ww21.4s, Ww23.4s\n"
654 "fmul U22.4s, scratch.4s, half.4s\n"
655 "fmla U22.4s, Ww22.4s, half.4s\n"
656 "str qU22, [%x[outptr4], %x[mstride1]]\n"
657 "fmul U23.4s, scratch.4s, half.4s\n"
658 "fmls U23.4s, Ww22.4s, half.4s\n"
659 "str qU23, [%x[outptr4], %x[mstride2]]\n"
660 "add %x[outptr4], %x[outptr4], #0x10\n"
661
662 "fadd scratch.4s, Ww31.4s, Ww33.4s\n"
663 "fmul U32.4s, scratch.4s, half.4s\n"
664 "fmla U32.4s, Ww32.4s, half.4s\n"
665 "str qU32, [%x[outptr8], %x[mstride1]]\n"
666 "fmul U33.4s, scratch.4s, half.4s\n"
667 "fmls U33.4s, Ww32.4s, half.4s\n"
668 "str qU33, [%x[outptr8], %x[mstride2]]\n"
669 "add %x[outptr8], %x[outptr8], #0x10\n"
670
671 "fadd scratch.4s, Ww41.4s, Ww43.4s\n"
672 "fmul U42.4s, scratch.4s, half.4s\n"
673 "fmla U42.4s, Ww42.4s, half.4s\n"
674 "str qU42, [%x[outptr12], %x[mstride1]]\n"
675 "fmul U43.4s, scratch.4s, half.4s\n"
676 "fmls U43.4s, Ww42.4s, half.4s\n"
677 "str qU43, [%x[outptr12], %x[mstride2]]\n"
678 "add %x[outptr12], %x[outptr12], #0x10\n"
679
680 "subs %x[n_remaining_channels], %x[n_remaining_channels], #4\n"
681 "bne 1b\n"
682
683 // Tail size 1
684 "2:"
685 // Load tile of the kernel
686 "ldr sw_11, [%x[inptr0]]\n"
687 "str sU11, [%x[outptr0]]\n"
688 "ldr sw_12, [%x[inptr0], %x[colstride1]]\n"
689 "ldr sw_13, [%x[inptr0], %x[colstride2]]\n"
690 "str sU14, [%x[outptr0], %x[mstride3]]\n"
691 "add %x[inptr0], %x[inptr0], #0x04\n"
692
693 "ldr sw_21, [%x[inptr1]]\n"
694 "ldr sw_22, [%x[inptr1], %x[colstride1]]\n"
695 "ldr sw_23, [%x[inptr1], %x[colstride2]]\n"
696 "add %x[inptr1], %x[inptr1], #0x04\n"
697
698 "ldr sw_31, [%x[inptr2]]\n"
699 "str sU41, [%x[outptr12]]\n"
700 "ldr sw_32, [%x[inptr2], %x[colstride1]]\n"
701 "ldr sw_33, [%x[inptr2], %x[colstride2]]\n"
702 "str sU44, [%x[outptr12], %x[mstride3]]\n"
703 "add %x[inptr2], %x[inptr2], #0x04\n"
704
705 // Compute 2nd and 3rd rows of Ww
706 "fadd scratch.2s, w_11.2s, w_31.2s\n"
707 "fmul Ww21.2s, scratch.2s, half.2s\n"
708 "fmla Ww21.2s, w_21.2s, half.2s\n"
709 "str sU21, [%x[outptr4]]\n"
710 "fmul Ww31.2s, scratch.2s, half.2s\n"
711 "fmls Ww31.2s, w_21.2s, half.2s\n"
712 "str sU31, [%x[outptr8]]\n"
713
714 "fadd scratch.2s, w_12.2s, w_32.2s\n"
715 "fmul Ww22.2s, scratch.2s, half.2s\n"
716 "fmla Ww22.2s, w_22.2s, half.2s\n"
717 "fmul Ww32.2s, scratch.2s, half.2s\n"
718 "fmls Ww32.2s, w_22.2s, half.2s\n"
719
720 "fadd scratch.2s, w_13.2s, w_33.2s\n"
721 "fmul Ww23.2s, scratch.2s, half.2s\n"
722 "fmla Ww23.2s, w_23.2s, half.2s\n"
723 "str sU24, [%x[outptr4], %x[mstride3]]\n"
724 "fmul Ww33.2s, scratch.2s, half.2s\n"
725 "fmls Ww33.2s, w_23.2s, half.2s\n"
726 "str sU34, [%x[outptr8], %x[mstride3]]\n"
727
728 // Compute and store U, only need to compute the 2nd and 3rd columns of
729 // U and update output pointers
730 "fadd scratch.2s, Ww11.2s, Ww13.2s\n"
731 "fmul U12.2s, scratch.2s, half.2s\n"
732 "fmla U12.2s, Ww12.2s, half.2s\n"
733 "str sU12, [%x[outptr0], %x[mstride1]]\n"
734 "fmul U13.2s, scratch.2s, half.2s\n"
735 "fmls U13.2s, Ww12.2s, half.2s\n"
736 "str sU13, [%x[outptr0], %x[mstride2]]\n"
737 "add %x[outptr0], %x[outptr0], #0x04\n"
738
739 "fadd scratch.2s, Ww21.2s, Ww23.2s\n"
740 "fmul U22.2s, scratch.2s, half.2s\n"
741 "fmla U22.2s, Ww22.2s, half.2s\n"
742 "str sU22, [%x[outptr4], %x[mstride1]]\n"
743 "fmul U23.2s, scratch.2s, half.2s\n"
744 "fmls U23.2s, Ww22.2s, half.2s\n"
745 "str sU23, [%x[outptr4], %x[mstride2]]\n"
746 "add %x[outptr4], %x[outptr4], #0x04\n"
747
748 "fadd scratch.2s, Ww31.2s, Ww33.2s\n"
749 "fmul U32.2s, scratch.2s, half.2s\n"
750 "fmla U32.2s, Ww32.2s, half.2s\n"
751 "str sU32, [%x[outptr8], %x[mstride1]]\n"
752 "fmul U33.2s, scratch.2s, half.2s\n"
753 "fmls U33.2s, Ww32.2s, half.2s\n"
754 "str sU33, [%x[outptr8], %x[mstride2]]\n"
755 "add %x[outptr8], %x[outptr8], #0x04\n"
756
757 "fadd scratch.2s, Ww41.2s, Ww43.2s\n"
758 "fmul U42.2s, scratch.2s, half.2s\n"
759 "fmla U42.2s, Ww42.2s, half.2s\n"
760 "str sU42, [%x[outptr12], %x[mstride1]]\n"
761 "fmul U43.2s, scratch.2s, half.2s\n"
762 "fmls U43.2s, Ww42.2s, half.2s\n"
763 "str sU43, [%x[outptr12], %x[mstride2]]\n"
764 "add %x[outptr12], %x[outptr12], #0x04\n"
765
766 // Clear aliases
767 ".unreq half\n"
768 ".unreq scratch\n"
769 ".unreq w_11\n" ".unreq qw_11\n" ".unreq sw_11\n"
770 ".unreq w_12\n" ".unreq qw_12\n" ".unreq sw_12\n"
771 ".unreq w_13\n" ".unreq qw_13\n" ".unreq sw_13\n"
772 ".unreq w_21\n" ".unreq qw_21\n" ".unreq sw_21\n"
773 ".unreq w_22\n" ".unreq qw_22\n" ".unreq sw_22\n"
774 ".unreq w_23\n" ".unreq qw_23\n" ".unreq sw_23\n"
775 ".unreq w_31\n" ".unreq qw_31\n" ".unreq sw_31\n"
776 ".unreq w_32\n" ".unreq qw_32\n" ".unreq sw_32\n"
777 ".unreq w_33\n" ".unreq qw_33\n" ".unreq sw_33\n"
778 ".unreq Ww11\n" ".unreq Ww12\n" ".unreq Ww13\n"
779 ".unreq Ww21\n" ".unreq Ww22\n" ".unreq Ww23\n"
780 ".unreq Ww31\n" ".unreq Ww32\n" ".unreq Ww33\n"
781 ".unreq Ww41\n" ".unreq Ww42\n" ".unreq Ww43\n"
782 ".unreq U11\n" ".unreq U12\n" ".unreq U13\n" ".unreq U14\n"
783 ".unreq U21\n" ".unreq U22\n" ".unreq U23\n" ".unreq U24\n"
784 ".unreq U31\n" ".unreq U32\n" ".unreq U33\n" ".unreq U34\n"
785 ".unreq U41\n" ".unreq U42\n" ".unreq U43\n" ".unreq U44\n"
786 ".unreq qU11\n" ".unreq qU12\n" ".unreq qU13\n" ".unreq qU14\n"
787 ".unreq qU21\n" ".unreq qU22\n" ".unreq qU23\n" ".unreq qU24\n"
788 ".unreq qU31\n" ".unreq qU32\n" ".unreq qU33\n" ".unreq qU34\n"
789 ".unreq qU41\n" ".unreq qU42\n" ".unreq qU43\n" ".unreq qU44\n"
790 ".unreq sU11\n" ".unreq sU12\n" ".unreq sU13\n" ".unreq sU14\n"
791 ".unreq sU21\n" ".unreq sU22\n" ".unreq sU23\n" ".unreq sU24\n"
792 ".unreq sU31\n" ".unreq sU32\n" ".unreq sU33\n" ".unreq sU34\n"
793 ".unreq sU41\n" ".unreq sU42\n" ".unreq sU43\n" ".unreq sU44\n"
794
795 : [inptr0] "+r" (inptr0),
796 [inptr1] "+r" (inptr1),
797 [inptr2] "+r" (inptr2),
798 [outptr0] "+r" (outptr0),
799 [outptr4] "+r" (outptr4),
800 [outptr8] "+r" (outptr8),
801 [outptr12] "+r" (outptr12),
802 [n_remaining_channels] "+r" (n_remaining_channels)
803 : [mstride1] "r" (sizeof(float) * mstride),
804 [mstride2] "r" (sizeof(float) * mstride * 2),
805 [mstride3] "r" (sizeof(float) * mstride * 3),
806 [colstride1] "r" (sizeof(float) * kernel_col_stride),
807 [colstride2] "r" (sizeof(float) * kernel_col_stride * 2),
808 [one_half] "r" (0.5f)
809 : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
810 "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
811 "v20", "v21", "v22", "v23", "v24"
812 );
813
814 // Progression to complete stride
815 outptr0 += matrix_row_stride - n_output_channels;
816 outptr4 += matrix_row_stride - n_output_channels;
817 outptr8 += matrix_row_stride - n_output_channels;
818 outptr12 += matrix_row_stride - n_output_channels;
819 }
820}
821}
822#endif // __aarch64__