Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / 3b9a564fd4573d7cf09e3203eb8a9a30fd5969c9 / . / src / core / CL / cl_kernels / load_store_utility.h
blob: cb833d016b994fe814a4a68dc0023b6857c9d29d [file] [log] [blame]
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]1/*
2 * Copyright (c) 2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24/** Partially store the 0 to (n-1)th rows of the given variables
25 * @name STORE_ROW_PARTIAL_n
26 * Within each row, store the lower @p STORE_N0 elements of vectors of width @p N0
27 *
28 * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
29 *
30 * @param[in] N0 The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16
31 * @param[in] STORE_N0 The **lower** size of the vectors to store. Supported: [1-16 and <= @p N0
32 * @param[in] DATA_TYPE The data type of the vectors
33 * @param[in] BASENAME The basename of the variables
34 * @param[in] PTR The base pointer
35 * @param[in] STRIDE_Y The stride value in y-axis direction
36 * @param[in] Z The offset in z-axis direction
37 * @{
38 */
39#define STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
40 VSTORE_PARTIAL(N0, STORE_N0) \
41 (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
42
43#define STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
44 STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
45 VSTORE_PARTIAL(N0, STORE_N0) \
46 (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
47
48#define STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
49 STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
50 VSTORE_PARTIAL(N0, STORE_N0) \
51 (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
52
53#define STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
54 STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
55 VSTORE_PARTIAL(N0, STORE_N0) \
56 (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
57
58#define STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
59 STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
60 VSTORE_PARTIAL(N0, STORE_N0) \
61 (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
62
63#define STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
64 STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
65 VSTORE_PARTIAL(N0, STORE_N0) \
66 (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
67
68#define STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
69 STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
70 VSTORE_PARTIAL(N0, STORE_N0) \
71 (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
72
73#define STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
74 STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
75 VSTORE_PARTIAL(N0, STORE_N0) \
76 (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
77
78#define STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
79 STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
80 VSTORE_PARTIAL(N0, STORE_N0) \
81 (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
82
83#define STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
84 STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
85 VSTORE_PARTIAL(N0, STORE_N0) \
86 (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
87
88#define STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
89 STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
90 VSTORE_PARTIAL(N0, STORE_N0) \
91 (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
92
93#define STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
94 STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
95 VSTORE_PARTIAL(N0, STORE_N0) \
96 (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
97
98#define STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
99 STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
100 VSTORE_PARTIAL(N0, STORE_N0) \
101 (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
102
103#define STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
104 STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
105 VSTORE_PARTIAL(N0, STORE_N0) \
106 (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
107
108#define STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
109 STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
110 VSTORE_PARTIAL(N0, STORE_N0) \
111 (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
112
113#define STORE_ROW_PARTIAL_16(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
114 STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
115 VSTORE_PARTIAL(N0, STORE_N0) \
116 (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
117/** @} */ // end of groupd STORE_ROW_PARTIAL_n
118
119/** Partially store a block of the given size STORE_M0xSTORE_N0
120 * @name STORE_BLOCK_PARTIAL
121 *
122 * @note The vector width @p N0 is also required for correct partial storing behaviour.
123 * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
124 *
125 * The data to store is expected to have consecutive names for each row.
126 * E.g., for STORE_M0=3 and basename=c, the expected names are c0, c1 and c2.
127 * The Z offset is expected to have consecutive names.
128 * E.g., for STORE_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
129 *
130 * @param[in] STORE_M0 The number of rows to store. Supported: 1-16
131 * @param[in] STORE_N0 The lower number of elements of vectors to store. Supported: 1-16 and <= @p N0
132 * @param[in] N0 The size of each vector. Supported: 1, 2, 3, 4, 8, 16
133 * @param[in] DATA_TYPE The data type of the vectors
134 * @param[in] BASENAME The basename of the variables
135 * @param[in] PTR The base pointer
136 * @param[in] STRIDE_Y The stride value in y-axis direction
137 * @param[in] Z The offset in z-axis direction
138 * @{
139 */
140#define STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_PARTIAL_##STORE_M0(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
141#define STORE_BLOCK_PARTIAL(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
142/** Store a block that can be partial in both x and y dimensions
143 *
144 * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
145 *
146 * The data to store is expected to have consecutive names for each row.
147 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
148 * The Z offset is expected to have consecutive names.
149 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
150 *
151 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
152 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
153 * @param[in] DATA_TYPE The data type of the vectors
154 * @param[in] BASENAME The basename of the variables
155 * @param[in] PTR The base pointer
156 * @param[in] STRIDE_Y The stride value in y-axis direction
157 * @param[in] Z The offset in z-axis direction
158 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
159 * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
160 * @param[in] N Total number of columns. Used to detect if current block is at the boundary in x.
161 * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
162 * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
163 */
164#define STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \
165 if(!(PARTIAL_COND_X) && !(PARTIAL_COND_Y)) \
166 { \
167 STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
168 } \
169 else if((PARTIAL_COND_Y) && !(PARTIAL_COND_X)) \
170 { \
171 STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
172 } \
173 else if(!(PARTIAL_COND_Y) && (PARTIAL_COND_X)) \
174 { \
175 STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
176 } \
177 else \
178 { \
179 STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
180 }
181/** Store a block that can only be partial in x but not y.
182 *
183 * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
184 *
185 * The data to store is expected to have consecutive names for each row.
186 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
187 * The Z offset is expected to have consecutive names.
188 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
189 *
190 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
191 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
192 * @param[in] DATA_TYPE The data type of the vectors
193 * @param[in] BASENAME The basename of the variables
194 * @param[in] PTR The base pointer
195 * @param[in] STRIDE_Y The stride value in y-axis direction
196 * @param[in] Z The offset in z-axis direction
197 * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
198 * @param[in] N Total number of columns. Used to detect if current block is at the boundary in x.
199 * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
200 */
201#define STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, N, PARTIAL_COND_X) \
202 if(!(PARTIAL_COND_X)) \
203 { \
204 STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
205 } \
206 else \
207 { \
208 STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
209 }
210/** Store a block that can only be partial in y but not x.
211 *
212 * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
213 *
214 * The data to store is expected to have consecutive names for each row.
215 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
216 * The Z offset is expected to have consecutive names.
217 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
218 *
219 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
220 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
221 * @param[in] DATA_TYPE The data type of the vectors
222 * @param[in] BASENAME The basename of the variables
223 * @param[in] PTR The base pointer
224 * @param[in] STRIDE_Y The stride value in y-axis direction
225 * @param[in] Z The offset in z-axis direction
226 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
227 * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
228 */
229#define STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) \
230 if(!(PARTIAL_COND_Y)) \
231 { \
232 STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
233 } \
234 else \
235 { \
236 STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \
237 }
238/** @} */ // end of group STORE_BLOCK_PARTIAL
239
240#if defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0)
241
242/** Boundary-aware GEMM block store
243 * @name STORE_BLOCK_BOUNDARY_AWARE
244 * This macro assumes the following schemes to achieve boundary-awareness:
245 * - Overlapping load in Y axis from lhs tensor. This implies lhs has no padding along y dim.
246 * - Non-Overlapping(normal) load from rhs tensor. This imples rhs can have paddings.
247 * - Overlapping load in Y axis from bias tensor. This implies rhs has no padding along y dim.
248 * The macro then ensures that the dst tensor can be stored without any paddings in both x and y dim.
249 *
250 * In the y dimension, we place the partial blocks **at the beginning** while in the x dimension, we place the partial
251 * blocks **at the end**.
252 * Say, the dst tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/
253 * "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters:
254 *
255 * *--x--> x == 0 x == 1
256 * | |<------------------------------N-------------------------->|
257 * y |<--------------N0------------->|<----PARTIAL_STORE_N0----->|
258 * | -------------#############################################################
259 * * | | |...............................|...........................|
260 * y == 0 | PAR_..._M0 |......Boundary block in y......|.Boundary block in x and y.|
261 * | | |...............................|...........................|
262 * M --#############################################################
263 * | | | |...........................|
264 * y == 1 | M0 | Non-boundary block |....Boundary block in x....|
265 * | | | |...........................|
266 * |------------#############################################################
267 *
268 * Then @p PARTIAL_STORE_M0 = M % M0 and @p PARTIAL_STORE_N0 = N % N0
269 *
270 * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
271 *
272 * It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension,
273 * and select corresponding store methods such that the boundary detection logic is only added when needed.
274 *
275 * The data to store is expected to have consecutive names for each row.
276 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
277 * The Z offset is expected to have consecutive names.
278 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
279 *
280 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
281 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
282 * @param[in] DATA_TYPE The data type of the vectors
283 * @param[in] BASENAME The basename of the variables
284 * @param[in] PTR The base pointer
285 * @param[in] STRIDE_Y The stride value in y-axis direction
286 * @param[in] Z The offset in z-axis direction
287 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0)
288 * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported: [0, @p N0)
289 * @param[in] N Total number of columns. Used to detect if current block is at the boundary in x.
290 * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
291 * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
292 * @{
293 */
294#if PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
295// Case1: No partial blocks in either x or y
296#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \
297 STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
298
299#elif PARTIAL_STORE_M0 > 0 && PARTIAL_STORE_N0 == 0
300// Case2: Partial blocks in y
301#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \
302 STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y)
303
304#elif PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 > 0
305// Case3: Partial blocks in x
306#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \
307 STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, N, PARTIAL_COND_X)
308
309#else // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
310// Case4: Partial blocks in both x and y
311#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \
312 STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X)
313
314#endif // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
315
316#else // defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0)
317
318#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \
319 STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
320
321#endif // defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0)
322/** @} */ // end of group STORE_BLOCK_BOUNDARY_AWARE
323
324#if defined(PARTIAL_STORE_M0)
325/** Compute the start m0 row (LHS, BIAS and DST) in a boundary-aware way so as to avoid padding
326 * @name COMPUTE_M0_START_ROW
327 * If there're any partial blocks in y dimension, they are placed at the beginning of the rows.
328 * This shift amount is added to all rows such that the partial block (at the beginning) overlaps with the subsequent
329 * blocks in the y dimension to avoid any padding.
330 * EG: M0=4, PARTIAL_STORE_M0=1:
331 * | Non-overlapping | +M0_ROW_SHIFT (Overlapping)
332 * block 0 (partial)| start row = 0 | start row = 0
333 * block 1 (full) | start row = 4 | start row = 1
334 * block 2 (full) | start row = 8 | start row = 5
335 *
336 * @param[in] y Global id of current block in y.
337 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
338 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0)
339 * @{
340 */
341#define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \
342 ((uint)(max(0, (int)(y * M0) - (int)((M0 - PARTIAL_STORE_M0) % M0))))
343#else // defined(PARTIAL_STORE_M0)
344#define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \
345 ((uint)(y * M0))
346#endif // defined(PARTIAL_STORE_M0)
347/** @} */ // end of group COMPUTE_M0_START_ROW
348
349/** Store a vector that can only be partial in x.
350 *
351 * @note in case @p vec_size or @p leftover != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
352 *
353 * The data to store is expected to end in a 0.
354 * E.g., for basename=c, the expected name is c0.
355 *
356 * @param[in] basename The name of the variable without trailing 0
357 * @param[in] data_type The data type of the vector
358 * @param[in] ptr The base pointer
359 * @param[in] vec_size The vector size if cond = false. Supported: 1, 2, 3, 4, 8, 16
360 * @param[in] leftover The vector size if cond = true. Supported range: [1, @p vec_size0)
361 * @param[in] cond Condition to select either vec_size0 or vec_size1
362 * @{
363 */
364#define STORE_VECTOR_SELECT(basename, data_type, ptr, vec_size, leftover, cond) \
365 STORE_BLOCK_PARTIAL_IN_X(1, vec_size, data_type, basename, ptr, 0, 0, leftover, 0, cond)
366/** @} */ // end of group STORE_VECTOR_SELECT