Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / 3a704ae94fc5cbfce1053886f543b31977f1774c / . / src / core / CL / cl_kernels / load_store_utility.h
blob: 4daf0adc89275c4815fad8206f4b0f70615b5c92 [file] [log] [blame]
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]1/*
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]2 * Copyright (c) 2020, 2023 Arm Limited.
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
Giorgio Arena1e2af2a2020-10-15 17:39:41 +0100[diff] [blame]24
25/** Store the 0 to (n-1)th rows of the given variables
26 * @name STORE_ROW_n
27 *
28 * @param[in] N0 The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16
29 * @param[in] DATA_TYPE The data type of the vectors
30 * @param[in] BASENAME The basename of the variables
31 * @param[in] PTR The base pointer
32 * @param[in] STRIDE_Y The stride value in y-axis direction
33 * @param[in] Z The offset in z-axis direction
34 * @{
35 */
36#define STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
37 VSTORE(N0) \
38 (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
39
40#define STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
41 STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
42 VSTORE(N0) \
43 (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
44
45#define STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
46 STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
47 VSTORE(N0) \
48 (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
49
50#define STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
51 STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
52 VSTORE(N0) \
53 (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
54
55#define STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
56 STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
57 VSTORE(N0) \
58 (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
59
60#define STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
61 STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
62 VSTORE(N0) \
63 (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
64
65#define STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
66 STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
67 VSTORE(N0) \
68 (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
69
70#define STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
71 STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
72 VSTORE(N0) \
73 (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
74
75#define STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
76 STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
77 VSTORE(N0) \
78 (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
79
80#define STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
81 STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
82 VSTORE(N0) \
83 (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
84
85#define STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
86 STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
87 VSTORE(N0) \
88 (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
89
90#define STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
91 STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
92 VSTORE(N0) \
93 (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
94
95#define STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
96 STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
97 VSTORE(N0) \
98 (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
99
100#define STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
101 STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
102 VSTORE(N0) \
103 (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
104
105#define STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
106 STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
107 VSTORE(N0) \
108 (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
109
110#define STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
111 STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
112 VSTORE(N0) \
113 (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
114/** @} */ // end of groupd STORE_ROW_n
115
116/** Convert and store the 0th to (n-1)th rows of the given variables
117 * @name CONVERT_STORE_ROW_n
118 *
119 * @param[in] N0 The size of the vectors
120 * @param[in] DATA_TYPE The data type of the vectors
121 * @param[in] BASENAME The basename of the variables
122 * @param[in] PTR The base pointer
123 * @param[in] STRIDE_Y The stride value in y-axis direction
124 * @param[in] Z The offset in z-axis direction
125 * @{
126 */
127#define CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
128 VSTORE(N0) \
129 (CONVERT_SAT((BASENAME##0), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
130
131#define CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
132 CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
133 VSTORE(N0) \
134 (CONVERT_SAT((BASENAME##1), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
135
136#define CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
137 CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
138 VSTORE(N0) \
139 (CONVERT_SAT((BASENAME##2), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
140
141#define CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
142 CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
143 VSTORE(N0) \
144 (CONVERT_SAT((BASENAME##3), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
145
146#define CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
147 CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
148 VSTORE(N0) \
149 (CONVERT_SAT((BASENAME##4), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
150
151#define CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
152 CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
153 VSTORE(N0) \
154 (CONVERT_SAT((BASENAME##5), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
155
156#define CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
157 CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
158 VSTORE(N0) \
159 (CONVERT_SAT((BASENAME##6), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
160
161#define CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
162 CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
163 VSTORE(N0) \
164 (CONVERT_SAT((BASENAME##7), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
165
166#define CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
167 CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
168 VSTORE(N0) \
169 (CONVERT_SAT((BASENAME##8), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
170
171#define CONVERT_STORE_ROW_10(N0, DATA, BASENAME, PTR, STRIDE_Y, Z) \
172 CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
173 VSTORE(N0) \
174 (CONVERT_SAT((BASENAME##9), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
175
176#define CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
177 CONVERT_STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
178 VSTORE(N0) \
179 (CONVERT_SAT((BASENAME##A), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
180
181#define CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
182 CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
183 VSTORE(N0) \
184 (CONVERT_SAT((BASENAME##B), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
185
186#define CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
187 CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
188 VSTORE(N0) \
189 (CONVERT_SAT((BASENAME##C), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
190
191#define CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
192 CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
193 VSTORE(N0) \
194 (CONVERT_SAT((BASENAME##D), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
195
196#define CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
197 CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
198 VSTORE(N0) \
199 (CONVERT_SAT((BASENAME##E), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
200
201#define CONVERT_STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
202 CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
203 VSTORE(N0) \
204 (CONVERT_SAT((BASENAME##F), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
205
206/** @} */ // end of groupd CONVERT_STORE_ROW_n
207
208/** Store a block of the given size M0xN0
209 * @name STORE_BLOCK
210 *
211 * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
212 * The data to store is expected to have consecutive names for each row.
213 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
214 * The Z offset is expected to have consecutive names.
215 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
216 *
217 * @param[in] M0 The number of rows to store
218 * @param[in] N0 The size of each vector
219 * @param[in] DATA_TYPE The data type of the vectors
220 * @param[in] BASENAME The basename of the variables
221 * @param[in] PTR The base pointer
222 * @param[in] STRIDE_Y The stride value in y-axis direction
223 * @param[in] Z The offset in z-axis direction
224 * @{
225 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]226#define STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
227 STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
228#define STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
229 STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Giorgio Arena1e2af2a2020-10-15 17:39:41 +0100[diff] [blame]230/** @} */ // end of group STORE_BLOCK
231
232/** Convert and store a block of the given size M0xN0
233 * @name CONVERT_STORE_BLOCK
234 *
235 * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
236 * The data to store is expected to have consecutive names for each row.
237 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
238 * The Z offset is expected to have consecutive names.
239 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
240 *
241 * @param[in] M0 The number of rows to store
242 * @param[in] N0 The size of each vector
243 * @param[in] DATA_TYPE The data type of the vectors
244 * @param[in] BASENAME The basename of the variables
245 * @param[in] PTR The base pointer
246 * @param[in] STRIDE_Y The stride value in y-axis direction
247 * @param[in] Z The offset in z-axis direction
248 * @{
249 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]250#define CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
251 CONVERT_STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
252#define CONVERT_STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
253 CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Giorgio Arena1e2af2a2020-10-15 17:39:41 +0100[diff] [blame]254/** @} */ // end of group CONVERT_STORE_BLOCK
255
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]256/** Partially store the 0 to (n-1)th rows of the given variables
257 * @name STORE_ROW_PARTIAL_n
258 * Within each row, store the lower @p STORE_N0 elements of vectors of width @p N0
259 *
260 * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
261 *
262 * @param[in] N0 The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16
263 * @param[in] STORE_N0 The **lower** size of the vectors to store. Supported: [1-16 and <= @p N0
264 * @param[in] DATA_TYPE The data type of the vectors
265 * @param[in] BASENAME The basename of the variables
266 * @param[in] PTR The base pointer
267 * @param[in] STRIDE_Y The stride value in y-axis direction
268 * @param[in] Z The offset in z-axis direction
269 * @{
270 */
271#define STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
272 VSTORE_PARTIAL(N0, STORE_N0) \
273 (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
274
275#define STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
276 STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
277 VSTORE_PARTIAL(N0, STORE_N0) \
278 (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
279
280#define STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
281 STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
282 VSTORE_PARTIAL(N0, STORE_N0) \
283 (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
284
285#define STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
286 STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
287 VSTORE_PARTIAL(N0, STORE_N0) \
288 (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
289
290#define STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
291 STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
292 VSTORE_PARTIAL(N0, STORE_N0) \
293 (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
294
295#define STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
296 STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
297 VSTORE_PARTIAL(N0, STORE_N0) \
298 (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
299
300#define STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
301 STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
302 VSTORE_PARTIAL(N0, STORE_N0) \
303 (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
304
305#define STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
306 STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
307 VSTORE_PARTIAL(N0, STORE_N0) \
308 (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
309
310#define STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
311 STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
312 VSTORE_PARTIAL(N0, STORE_N0) \
313 (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
314
315#define STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
316 STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
317 VSTORE_PARTIAL(N0, STORE_N0) \
318 (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
319
320#define STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
321 STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
322 VSTORE_PARTIAL(N0, STORE_N0) \
323 (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
324
325#define STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
326 STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
327 VSTORE_PARTIAL(N0, STORE_N0) \
328 (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
329
330#define STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
331 STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
332 VSTORE_PARTIAL(N0, STORE_N0) \
333 (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
334
335#define STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
336 STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
337 VSTORE_PARTIAL(N0, STORE_N0) \
338 (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
339
340#define STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
341 STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
342 VSTORE_PARTIAL(N0, STORE_N0) \
343 (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
344
345#define STORE_ROW_PARTIAL_16(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
346 STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
347 VSTORE_PARTIAL(N0, STORE_N0) \
348 (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
349/** @} */ // end of groupd STORE_ROW_PARTIAL_n
350
351/** Partially store a block of the given size STORE_M0xSTORE_N0
352 * @name STORE_BLOCK_PARTIAL
353 *
354 * @note The vector width @p N0 is also required for correct partial storing behaviour.
355 * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
356 *
357 * The data to store is expected to have consecutive names for each row.
358 * E.g., for STORE_M0=3 and basename=c, the expected names are c0, c1 and c2.
359 * The Z offset is expected to have consecutive names.
360 * E.g., for STORE_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
361 *
362 * @param[in] STORE_M0 The number of rows to store. Supported: 1-16
363 * @param[in] STORE_N0 The lower number of elements of vectors to store. Supported: 1-16 and <= @p N0
364 * @param[in] N0 The size of each vector. Supported: 1, 2, 3, 4, 8, 16
365 * @param[in] DATA_TYPE The data type of the vectors
366 * @param[in] BASENAME The basename of the variables
367 * @param[in] PTR The base pointer
368 * @param[in] STRIDE_Y The stride value in y-axis direction
369 * @param[in] Z The offset in z-axis direction
370 * @{
371 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]372#define STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
373 STORE_ROW_PARTIAL_##STORE_M0(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
374#define STORE_BLOCK_PARTIAL(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
375 STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]376/** Store a block that can be partial in both x and y dimensions
377 *
378 * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
379 *
380 * The data to store is expected to have consecutive names for each row.
381 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
382 * The Z offset is expected to have consecutive names.
383 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
384 *
385 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
386 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
387 * @param[in] DATA_TYPE The data type of the vectors
388 * @param[in] BASENAME The basename of the variables
389 * @param[in] PTR The base pointer
390 * @param[in] STRIDE_Y The stride value in y-axis direction
391 * @param[in] Z The offset in z-axis direction
392 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
393 * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]394 * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
395 * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
396 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]397#define STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, \
398 PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
399 if (!(PARTIAL_COND_X) && !(PARTIAL_COND_Y)) \
400 { \
401 STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
402 } \
403 else if ((PARTIAL_COND_Y) && !(PARTIAL_COND_X)) \
404 { \
405 STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
406 } \
407 else if (!(PARTIAL_COND_Y) && (PARTIAL_COND_X)) \
408 { \
409 STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
410 } \
411 else \
412 { \
413 STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]414 }
415/** Store a block that can only be partial in x but not y.
416 *
417 * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
418 *
419 * The data to store is expected to have consecutive names for each row.
420 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
421 * The Z offset is expected to have consecutive names.
422 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
423 *
424 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
425 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
426 * @param[in] DATA_TYPE The data type of the vectors
427 * @param[in] BASENAME The basename of the variables
428 * @param[in] PTR The base pointer
429 * @param[in] STRIDE_Y The stride value in y-axis direction
430 * @param[in] Z The offset in z-axis direction
431 * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]432 * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
433 */
Giorgio Arena1e2af2a2020-10-15 17:39:41 +0100[diff] [blame]434#define STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X) \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]435 if (!(PARTIAL_COND_X)) \
Giorgio Arena1e2af2a2020-10-15 17:39:41 +0100[diff] [blame]436 { \
437 STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
438 } \
439 else \
440 { \
441 STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]442 }
443/** Store a block that can only be partial in y but not x.
444 *
445 * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
446 *
447 * The data to store is expected to have consecutive names for each row.
448 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
449 * The Z offset is expected to have consecutive names.
450 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
451 *
452 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
453 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
454 * @param[in] DATA_TYPE The data type of the vectors
455 * @param[in] BASENAME The basename of the variables
456 * @param[in] PTR The base pointer
457 * @param[in] STRIDE_Y The stride value in y-axis direction
458 * @param[in] Z The offset in z-axis direction
459 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
460 * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
461 */
462#define STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]463 if (!(PARTIAL_COND_Y)) \
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]464 { \
465 STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
466 } \
467 else \
468 { \
469 STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
470 }
471/** @} */ // end of group STORE_BLOCK_PARTIAL
472
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]473/** Boundary-aware GEMM block store
474 * @name STORE_BLOCK_BOUNDARY_AWARE
475 * This macro assumes the following schemes to achieve boundary-awareness:
476 * - Overlapping load in Y axis from lhs tensor. This implies lhs has no padding along y dim.
477 * - Non-Overlapping(normal) load from rhs tensor. This imples rhs can have paddings.
478 * - Overlapping load in Y axis from bias tensor. This implies rhs has no padding along y dim.
479 * The macro then ensures that the dst tensor can be stored without any paddings in both x and y dim.
480 *
481 * In the y dimension, we place the partial blocks **at the beginning** while in the x dimension, we place the partial
482 * blocks **at the end**.
483 * Say, the dst tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/
484 * "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters:
485 *
486 * *--x--> x == 0 x == 1
487 * | |<------------------------------N-------------------------->|
488 * y |<--------------N0------------->|<----PARTIAL_STORE_N0----->|
489 * | -------------#############################################################
490 * * | | |...............................|...........................|
491 * y == 0 | PAR_..._M0 |......Boundary block in y......|.Boundary block in x and y.|
492 * | | |...............................|...........................|
493 * M --#############################################################
494 * | | | |...........................|
495 * y == 1 | M0 | Non-boundary block |....Boundary block in x....|
496 * | | | |...........................|
497 * |------------#############################################################
498 *
499 * Then @p PARTIAL_STORE_M0 = M % M0 and @p PARTIAL_STORE_N0 = N % N0
500 *
501 * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
502 *
503 * It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension,
504 * and select corresponding store methods such that the boundary detection logic is only added when needed.
505 *
506 * The data to store is expected to have consecutive names for each row.
507 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
508 * The Z offset is expected to have consecutive names.
509 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
510 *
511 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
512 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
513 * @param[in] DATA_TYPE The data type of the vectors
514 * @param[in] BASENAME The basename of the variables
515 * @param[in] PTR The base pointer
516 * @param[in] STRIDE_Y The stride value in y-axis direction
517 * @param[in] Z The offset in z-axis direction
518 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0)
519 * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported: [0, @p N0)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]520 * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
521 * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
522 * @{
523 */
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]524#if defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]525#if PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
526// Case1: No partial blocks in either x or y
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]527#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, \
528 PARTIAL_COND_Y, PARTIAL_COND_X) \
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]529 STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
530
531#elif PARTIAL_STORE_M0 > 0 && PARTIAL_STORE_N0 == 0
532// Case2: Partial blocks in y
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]533#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, \
534 PARTIAL_COND_Y, PARTIAL_COND_X) \
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]535 STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y)
536
537#elif PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 > 0
538// Case3: Partial blocks in x
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]539#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, \
540 PARTIAL_COND_Y, PARTIAL_COND_X) \
Giorgio Arena1e2af2a2020-10-15 17:39:41 +0100[diff] [blame]541 STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]542
543#else // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
544// Case4: Partial blocks in both x and y
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]545#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, \
546 PARTIAL_COND_Y, PARTIAL_COND_X) \
547 STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, \
548 PARTIAL_COND_Y, PARTIAL_COND_X)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]549
550#endif // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
551
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]552#endif // defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0)
553/** @} */ // end of group STORE_BLOCK_BOUNDARY_AWARE
554
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]555/** Compute the start m0 row (LHS, BIAS and DST) in a boundary-aware way so as to avoid padding
556 * @name COMPUTE_M0_START_ROW
557 * If there're any partial blocks in y dimension, they are placed at the beginning of the rows.
558 * This shift amount is added to all rows such that the partial block (at the beginning) overlaps with the subsequent
559 * blocks in the y dimension to avoid any padding.
560 * EG: M0=4, PARTIAL_STORE_M0=1:
561 * | Non-overlapping | +M0_ROW_SHIFT (Overlapping)
562 * block 0 (partial)| start row = 0 | start row = 0
563 * block 1 (full) | start row = 4 | start row = 1
564 * block 2 (full) | start row = 8 | start row = 5
565 *
566 * @param[in] y Global id of current block in y.
567 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
568 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0)
569 * @{
570 */
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]571#if defined(PARTIAL_STORE_M0)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]572#define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \
573 ((uint)(max(0, (int)(y * M0) - (int)((M0 - PARTIAL_STORE_M0) % M0))))
574#else // defined(PARTIAL_STORE_M0)
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]575#define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) ((uint)(y * M0))
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]576#endif // defined(PARTIAL_STORE_M0)
577/** @} */ // end of group COMPUTE_M0_START_ROW
578
579/** Store a vector that can only be partial in x.
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]580 * @name STORE_VECTOR_SELECT
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]581 * @note in case @p vec_size or @p leftover != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
582 *
583 * The data to store is expected to end in a 0.
584 * E.g., for basename=c, the expected name is c0.
585 *
586 * @param[in] basename The name of the variable without trailing 0
587 * @param[in] data_type The data type of the vector
588 * @param[in] ptr The base pointer
589 * @param[in] vec_size The vector size if cond = false. Supported: 1, 2, 3, 4, 8, 16
590 * @param[in] leftover The vector size if cond = true. Supported range: [1, @p vec_size0)
591 * @param[in] cond Condition to select either vec_size0 or vec_size1
592 * @{
593 */
594#define STORE_VECTOR_SELECT(basename, data_type, ptr, vec_size, leftover, cond) \
Giorgio Arena1e2af2a2020-10-15 17:39:41 +0100[diff] [blame]595 STORE_BLOCK_PARTIAL_IN_X(1, vec_size, data_type, basename, ptr, 0, 0, leftover, cond)
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]596/** @} */ // end of group STORE_VECTOR_SELECT