Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / fdf56fb9d414a754e7cedfdc1351ab0ce2866a0c / . / src / core / CL / cl_kernels / gemm_helpers.h
blob: 4bef02314f1edc3ec639617e2f221cb1c8973138 [file] [log] [blame]
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1/*
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]2 * Copyright (c) 2019-2021, 2023 Arm Limited.
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]24#include "activation_float_helpers.h"
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]25#include "helpers.h"
26
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]27/** Utility macro to access a vector with the scalar positions
28 *
29 * Supported cases are: Offset can only be of the same size of the OpenCL vector (2,3,4,8,16)
30 *
31 * @param[in] offset The offset within the vector. Offset can only be of the same size of the OpenCL vector (2,3,4,8,16)
32 * @param[in] n0 The number of consecutive columns to access. n0 + offset must be <= 16
33 * @param[in] x Vector to access
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]34 *
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]35 */
36#define SCALAR_ACCESS_STR(offset, n0, x) scalar_access_##offset##_##n0(x)
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]37#define SCALAR_ACCESS(offset, n0, x) SCALAR_ACCESS_STR(offset, n0, x)
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]38
39// offset == 0
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]40#define scalar_access_0_1(x) ((x).s0)
41#define scalar_access_0_2(x) ((x).s01)
42#define scalar_access_0_3(x) ((x).s012)
43#define scalar_access_0_4(x) ((x).s0123)
44#define scalar_access_0_8(x) ((x).s01234567)
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]45#define scalar_access_0_16(x) ((x).s0123456789ABCDEF)
46
47// offset == 1
48#define scalar_access_1_1(x) ((x).s1)
49#define scalar_access_1_2(x) ((x).s12)
50#define scalar_access_1_3(x) ((x).s123)
51#define scalar_access_1_4(x) ((x).s1234)
52#define scalar_access_1_8(x) ((x).s12345678)
53
54// offset == 2
55#define scalar_access_2_1(x) ((x).s2)
56#define scalar_access_2_2(x) ((x).s23)
57#define scalar_access_2_3(x) ((x).s234)
58#define scalar_access_2_4(x) ((x).s2345)
59#define scalar_access_2_8(x) ((x).s23456789)
60
61// offset == 3
62#define scalar_access_3_1(x) ((x).s3)
63#define scalar_access_3_2(x) ((x).s34)
64#define scalar_access_3_3(x) ((x).s345)
65#define scalar_access_3_4(x) ((x).s3456)
66#define scalar_access_3_8(x) ((x).s3456789A)
67
68// offset == 4
69#define scalar_access_4_1(x) ((x).s4)
70#define scalar_access_4_2(x) ((x).s45)
71#define scalar_access_4_3(x) ((x).s456)
72#define scalar_access_4_4(x) ((x).s4567)
73#define scalar_access_4_8(x) ((x).s456789AB)
74
75// offset == 8
76#define scalar_access_8_1(x) ((x).s8)
77#define scalar_access_8_2(x) ((x).s89)
78#define scalar_access_8_3(x) ((x).s89A)
79#define scalar_access_8_4(x) ((x).s89AB)
80#define scalar_access_8_8(x) ((x).s89ABCDEF)
81
82// offset == 12
83#define scalar_access_12_1(x) ((x).sC)
84#define scalar_access_12_2(x) ((x).sCD)
85#define scalar_access_12_3(x) ((x).sCDE)
86#define scalar_access_12_4(x) ((x).sCDEF)
87
88// offset == 16
89#define scalar_access_16_1(x) ((x).sF)
90
91/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1) without allocating variables.
92 * @name LOAD_TENSOR_ROW_n
93 *
94 * @param[in] N0 The number of columns to load
95 * @param[in] DATA_TYPE The data type of variables
96 * @param[in] BASENAME The basename of the destination variables for the loaded rows
97 * @param[in] PTR The base pointer
98 * @param[in] COL_OFFSET The column vector offset. COL_OFFSET + N0 must be <= 16
99 * @param[in] STRIDE_Y The stride value in y-axis direction
100 * @param[in] Z The z-axis offset vector
101 * @{
102 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]103#define LOAD_TENSOR_ROW_0(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) ({})
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]104
105#define LOAD_TENSOR_ROW_1(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
106 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##0) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
107
108#define LOAD_TENSOR_ROW_2(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
109 LOAD_TENSOR_ROW_1(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
110 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##1) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
111
112#define LOAD_TENSOR_ROW_3(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
113 LOAD_TENSOR_ROW_2(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
114 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##2) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
115
116#define LOAD_TENSOR_ROW_4(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
117 LOAD_TENSOR_ROW_3(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
118 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##3) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
119
120#define LOAD_TENSOR_ROW_5(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
121 LOAD_TENSOR_ROW_4(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
122 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##4) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
123
124#define LOAD_TENSOR_ROW_6(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
125 LOAD_TENSOR_ROW_5(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
126 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##5) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
127
128#define LOAD_TENSOR_ROW_7(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
129 LOAD_TENSOR_ROW_6(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
130 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##6) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
131
132#define LOAD_TENSOR_ROW_8(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
133 LOAD_TENSOR_ROW_7(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
134 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##7) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
135
136#define LOAD_TENSOR_ROW_9(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
137 LOAD_TENSOR_ROW_8(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
138 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##8) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
139
140#define LOAD_TENSOR_ROW_10(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
141 LOAD_TENSOR_ROW_9(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
142 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##9) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
143
144#define LOAD_TENSOR_ROW_11(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
145 LOAD_TENSOR_ROW_10(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
146 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##A) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
147
148#define LOAD_TENSOR_ROW_12(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
149 LOAD_TENSOR_ROW_11(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
150 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##B) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
151
152#define LOAD_TENSOR_ROW_13(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
153 LOAD_TENSOR_ROW_12(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
154 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##C) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
155
156#define LOAD_TENSOR_ROW_14(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
157 LOAD_TENSOR_ROW_13(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
158 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##D) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
159
160#define LOAD_TENSOR_ROW_15(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
161 LOAD_TENSOR_ROW_14(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
162 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##E) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
163
164#define LOAD_TENSOR_ROW_16(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
165 LOAD_TENSOR_ROW_15(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
166 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##F) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
167/** @}*/ // end of group LOAD_TENSOR_ROW_n
168
169/** Load tensor (consecutive rows and columns) with Z offset.
170 * @name LOAD_TENSOR
171 *
172 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
173 * The data to load is expected to have consecutive names for each row.
174 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
175 * The Z offset is expected to have consecutive names.
176 * E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
177 *
178 * @param[in] M0 The number of consecutive rows
179 * @param[in] N0 The number of consecutive columns
180 * @param[in] DATA_TYPE The data type of the target
181 * @param[in] BASENAME The basename of the result variables
182 * @param[in] PTR The base pointer for the data
183 * @param[in] COL_OFFSET The column vector offset. COL_OFFSET + N0 must be <= 16
184 * @param[in] STRIDE_Y The stride in y-axis direction
185 * @param[in] Z The z-axis offset vector
186 * @{
187 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]188#define LOAD_TENSOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
189 LOAD_TENSOR_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z)
190#define LOAD_TENSOR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
191 LOAD_TENSOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z)
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]192/** @} */ // end of group LOAD_TENSOR
193
194/** Load 2D tensor (consecutive rows and columns) with Z offset.
195 * @name LOAD_TENSOR_M0Xn
196 *
197 * @param[in] M0 The number of rows to load [0-16]
198 * @param[in] N0 The number of columns to load [0-16]
199 * @param[in] DATA_TYPE The data type of variables
200 * @param[in] BASENAME The basename of the destination variables for the loaded rows
201 * @param[in] PTR The base pointer
202 * @param[in] STRIDE_Y The stride value in y-axis direction
203 * @param[in] Z The z-axis offset vector
204 * @{
205 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]206#define LOAD_TENSOR_M0X0(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) ({})
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]207
208#define LOAD_TENSOR_M0X1(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
209 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
210
211#define LOAD_TENSOR_M0X2(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
212 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
213
214#define LOAD_TENSOR_M0X3(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
215 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
216
217#define LOAD_TENSOR_M0X4(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
218 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
219
220#define LOAD_TENSOR_M0X5(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
221 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
222 LOAD_TENSOR(M0, 1, DATA_TYPE, a, input_ptr + 4 * sizeof(DATA_TYPE), 4, src_stride_y, zin);
223
224#define LOAD_TENSOR_M0X6(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
225 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
226 LOAD_TENSOR(M0, 2, DATA_TYPE, a, input_ptr + 4 * sizeof(DATA_TYPE), 4, src_stride_y, zin);
227
228#define LOAD_TENSOR_M0X7(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
229 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
230 LOAD_TENSOR(M0, 3, DATA_TYPE, a, input_ptr + 4 * sizeof(DATA_TYPE), 4, src_stride_y, zin);
231
232#define LOAD_TENSOR_M0X8(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
233 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
234
235#define LOAD_TENSOR_M0X9(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
236 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr 0, src_stride_y, zin); \
237 LOAD_TENSOR(M0, 1, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
238
239#define LOAD_TENSOR_M0X10(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
240 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
241 LOAD_TENSOR(M0, 2, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
242
243#define LOAD_TENSOR_M0X11(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
244 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
245 LOAD_TENSOR(M0, 3, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
246
247#define LOAD_TENSOR_M0X12(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
248 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
249 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
250
251#define LOAD_TENSOR_M0X13(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
252 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
253 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin); \
254 LOAD_TENSOR(M0, 1, DATA_TYPE, a, input_ptr + 12 * sizeof(DATA_TYPE), 12, src_stride_y, zin);
255
256#define LOAD_TENSOR_M0X14(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
257 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr 0, src_stride_y, zin); \
258 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin); \
259 LOAD_TENSOR(M0, 2, DATA_TYPE, a, input_ptr + 12 * sizeof(DATA_TYPE), 12, src_stride_y, zin);
260
261#define LOAD_TENSOR_M0X15(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
262 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
263 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin); \
264 LOAD_TENSOR(M0, 3, DATA_TYPE, a, input_ptr + 12 * sizeof(DATA_TYPE), 12, src_stride_y, zin);
265
266#define LOAD_TENSOR_M0X16(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
267 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
268/** @}*/ // end of group LOAD_TENSOR_M0Xn
269
270/** Load 2D tensor (consecutive rows and columns) with Z offset.
271 * @name LOAD_TENSOR_M0XN0
272 *
273 * @param[in] M0 The number of consecutive rows [0-16]
274 * @param[in] N0 The number of consecutive columns [0-16]
275 * @param[in] DATA_TYPE The data type of the target
276 * @param[in] BASENAME The basename of the result variables
277 * @param[in] PTR The base pointer for the data
278 * @param[in] STRIDE_Y The stride in y-axis direction
279 * @param[in] Z The z-axis offset vector
280 * @{
281 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]282#define LOAD_TENSOR_M0XN0_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
283 LOAD_TENSOR_M0X##N0(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
284#define LOAD_TENSOR_M0XN0(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
285 LOAD_TENSOR_M0XN0_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]286/** @}*/ // end of group LOAD_TENSOR_M0XN0
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]287
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]288/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
289 * @name LOAD_ROW_n
290 *
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]291 * @param[in] N0 The number of columns to load
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]292 * @param[in] DATA_TYPE The data type of variables
293 * @param[in] BASENAME The basename of the destination variables for the loaded rows
294 * @param[in] PTR The base pointer
295 * @param[in] OFFSET The offset within a row
296 * @param[in] STRIDE_Y The stride value in y-axis direction
297 * @param[in] Z The z-axis offset vector
298 * @{
299 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]300#define LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
301 VEC_DATA_TYPE(DATA_TYPE, N0) \
302 BASENAME##0 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y + Z##0));
303
304#define LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
305 LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
306 VEC_DATA_TYPE(DATA_TYPE, N0) \
307 BASENAME##1 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y + Z##1));
308
309#define LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
310 LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
311 VEC_DATA_TYPE(DATA_TYPE, N0) \
312 BASENAME##2 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y + Z##2));
313
314#define LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
315 LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
316 VEC_DATA_TYPE(DATA_TYPE, N0) \
317 BASENAME##3 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y + Z##3));
318
319#define LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
320 LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
321 VEC_DATA_TYPE(DATA_TYPE, N0) \
322 BASENAME##4 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y + Z##4));
323
324#define LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
325 LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
326 VEC_DATA_TYPE(DATA_TYPE, N0) \
327 BASENAME##5 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y + Z##5));
328
329#define LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
330 LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
331 VEC_DATA_TYPE(DATA_TYPE, N0) \
332 BASENAME##6 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y + Z##6));
333
334#define LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
335 LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
336 VEC_DATA_TYPE(DATA_TYPE, N0) \
337 BASENAME##7 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y + Z##7));
338
339#define LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
340 LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
341 VEC_DATA_TYPE(DATA_TYPE, N0) \
342 BASENAME##8 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y + Z##8));
343
344#define LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
345 LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
346 VEC_DATA_TYPE(DATA_TYPE, N0) \
347 BASENAME##9 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y + Z##9));
348
349#define LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
350 LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
351 VEC_DATA_TYPE(DATA_TYPE, N0) \
352 BASENAME##A = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y + Z##A));
353
354#define LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
355 LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
356 VEC_DATA_TYPE(DATA_TYPE, N0) \
357 BASENAME##B = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y + Z##B));
358
359#define LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
360 LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
361 VEC_DATA_TYPE(DATA_TYPE, N0) \
362 BASENAME##C = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y + Z##C));
363
364#define LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
365 LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
366 VEC_DATA_TYPE(DATA_TYPE, N0) \
367 BASENAME##D = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y + Z##D));
368
369#define LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
370 LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
371 VEC_DATA_TYPE(DATA_TYPE, N0) \
372 BASENAME##E = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y + Z##E));
373
374#define LOAD_ROW_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
375 LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
376 VEC_DATA_TYPE(DATA_TYPE, N0) \
377 BASENAME##F = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y + Z##F));
378
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]379/** @}*/ // end of group LOAD_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]380
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]381/** Load Blocks (consecutive rows and columns) with Z offset.
382 * @name LOAD_BLOCK
383 *
384 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
385 * The data to load is expected to have consecutive names for each row.
386 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
387 * The Z offset is expected to have consecutive names.
388 * E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
389 *
390 * @param[in] M0 The number of consecutive rows
391 * @param[in] N0 The number of consecutive columns
392 * @param[in] DATA_TYPE The data type of the target
393 * @param[in] BASENAME The basename of the result variables
394 * @param[in] PTR The base pointer for the data
395 * @param[in] OFFSET The offset within a row
396 * @param[in] STRIDE_Y The stride in y-axis direction
397 * @param[in] Z The z-axis offset vector
398 * @{
399 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]400#define LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
401 LOAD_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
402#define LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
403 LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]404/** @} */ // end of group LOAD_BLOCK
405
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]406/** Partially load the 0 to (n-1)th rows of the given variables
407 * @name LOAD_ROW_PARTIAL_n
408 * Within each row, load the lower @p LOAD_N0 elements of vectors of width @p N0
409 *
410 * @note in case @p LOAD_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty.
411 *
412 * @param[in] N0 The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16
413 * @param[in] LOAD_N0 The **lower** size of the vectors to load. Supported: [1-16 and <= @p N0
414 * @param[in] DATA_TYPE The data type of the vectors
415 * @param[in] BASENAME The basename of the variables
416 * @param[in] PTR The base pointer
417 * @param[in] OFFSET The offset within a row
418 * @param[in] STRIDE_Y The stride value in y-axis direction
419 * @param[in] Z The offset in z-axis direction
420 * @{
421 */
422#define LOAD_ROW_PARTIAL_1(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
423 VLOAD_PARTIAL(N0, LOAD_N0) \
424 (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y + Z##0));
425
426#define LOAD_ROW_PARTIAL_2(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
427 LOAD_ROW_PARTIAL_1(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
428 VLOAD_PARTIAL(N0, LOAD_N0) \
429 (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y + Z##1));
430
431#define LOAD_ROW_PARTIAL_3(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
432 LOAD_ROW_PARTIAL_2(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
433 VLOAD_PARTIAL(N0, LOAD_N0) \
434 (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y + Z##2));
435
436#define LOAD_ROW_PARTIAL_4(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
437 LOAD_ROW_PARTIAL_3(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
438 VLOAD_PARTIAL(N0, LOAD_N0) \
439 (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y + Z##3));
440
441#define LOAD_ROW_PARTIAL_5(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
442 LOAD_ROW_PARTIAL_4(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
443 VLOAD_PARTIAL(N0, LOAD_N0) \
444 (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y + Z##4));
445
446#define LOAD_ROW_PARTIAL_6(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
447 LOAD_ROW_PARTIAL_5(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
448 VLOAD_PARTIAL(N0, LOAD_N0) \
449 (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y + Z##5));
450
451#define LOAD_ROW_PARTIAL_7(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
452 LOAD_ROW_PARTIAL_6(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
453 VLOAD_PARTIAL(N0, LOAD_N0) \
454 (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y + Z##6));
455
456#define LOAD_ROW_PARTIAL_8(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
457 LOAD_ROW_PARTIAL_7(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
458 VLOAD_PARTIAL(N0, LOAD_N0) \
459 (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y + Z##7));
460
461#define LOAD_ROW_PARTIAL_9(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
462 LOAD_ROW_PARTIAL_8(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
463 VLOAD_PARTIAL(N0, LOAD_N0) \
464 (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y + Z##8));
465
466#define LOAD_ROW_PARTIAL_10(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
467 LOAD_ROW_PARTIAL_9(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
468 VLOAD_PARTIAL(N0, LOAD_N0) \
469 (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y + Z##9));
470
471#define LOAD_ROW_PARTIAL_11(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
472 LOAD_ROW_PARTIAL_10(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
473 VLOAD_PARTIAL(N0, LOAD_N0) \
474 (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y + Z##A));
475
476#define LOAD_ROW_PARTIAL_12(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
477 LOAD_ROW_PARTIAL_11(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
478 VLOAD_PARTIAL(N0, LOAD_N0) \
479 (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y + Z##B));
480
481#define LOAD_ROW_PARTIAL_13(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
482 LOAD_ROW_PARTIAL_12(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
483 VLOAD_PARTIAL(N0, LOAD_N0) \
484 (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y + Z##C));
485
486#define LOAD_ROW_PARTIAL_14(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
487 LOAD_ROW_PARTIAL_13(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
488 VLOAD_PARTIAL(N0, LOAD_N0) \
489 (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y + Z##D));
490
491#define LOAD_ROW_PARTIAL_15(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
492 LOAD_ROW_PARTIAL_14(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
493 VLOAD_PARTIAL(N0, LOAD_N0) \
494 (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y + Z##E));
495
496#define LOAD_ROW_PARTIAL_16(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
497 LOAD_ROW_PARTIAL_15(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
498 VLOAD_PARTIAL(N0, LOAD_N0) \
499 (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y + Z##F));
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]500/** @} */ // end of group LOAD_ROW_PARTIAL_n
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]501
502/** Partially load a block of the given size LOAD_M0xLOAD_N0
503 * @name LOAD_BLOCK_PARTIAL
504 *
505 * @note The vector width @p N0 is also required for correct partial storing behaviour.
506 * @note in case @p LOAD_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty.
507 *
508 * The data to load is expected to have consecutive names for each row.
509 * E.g., for LOAD_M0=3 and basename=c, the expected names are c0, c1 and c2.
510 * The Z offset is expected to have consecutive names.
511 * E.g., for LOAD_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
512 *
513 * @param[in] LOAD_M0 The number of rows to load. Supported: 1-16
514 * @param[in] LOAD_N0 The lower number of elements of vectors to load. Supported: 1-16 and <= @p N0
515 * @param[in] N0 The size of each vector. Supported: 1, 2, 3, 4, 8, 16
516 * @param[in] DATA_TYPE The data type of the vectors
517 * @param[in] BASENAME The basename of the variables
518 * @param[in] PTR The base pointer
519 * @param[in] OFFSET The offset within a row
520 * @param[in] STRIDE_Y The stride value in y-axis direction
521 * @param[in] Z The offset in z-axis direction
522 * @{
523 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]524#define LOAD_BLOCK_PARTIAL_STR(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
525 LOAD_ROW_PARTIAL_##LOAD_M0(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
526#define LOAD_BLOCK_PARTIAL(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
527 LOAD_BLOCK_PARTIAL_STR(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]528/** Load a block that can be partial in both x and y dimensions
529 *
530 * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty.
531 *
532 * The data to load is expected to have consecutive names for each row.
533 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
534 * The Z offset is expected to have consecutive names.
535 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
536 *
537 * @param[in] M0 The number of rows to load, for non-partial blocks. Supported: 1-16
538 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
539 * @param[in] DATA_TYPE The data type of the vectors
540 * @param[in] BASENAME The basename of the variables
541 * @param[in] PTR The base pointer
542 * @param[in] OFFSET The offset within a row
543 * @param[in] STRIDE_Y The stride value in y-axis direction
544 * @param[in] Z The offset in z-axis direction
545 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
546 * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
547 * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial load Y. True to use PARTIAL_STORE_M0 rather than M0.
548 * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0.
549 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]550#define LOAD_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, \
551 PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
552 if (!(PARTIAL_COND_X) && !(PARTIAL_COND_Y)) \
553 { \
554 LOAD_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \
555 } \
556 else if ((PARTIAL_COND_Y) && !(PARTIAL_COND_X)) \
557 { \
558 LOAD_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \
559 } \
560 else if (!(PARTIAL_COND_Y) && (PARTIAL_COND_X)) \
561 { \
562 LOAD_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \
563 } \
564 else \
565 { \
566 LOAD_BLOCK_PARTIAL(PARTIAL_STORE_M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]567 }
568/** Load a block that can only be partial in x but not y.
569 *
570 * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty.
571 *
572 * The data to load is expected to have consecutive names for each row.
573 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
574 * The Z offset is expected to have consecutive names.
575 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
576 *
577 * @param[in] M0 The number of rows to load, for non-partial blocks. Supported: 1-16
578 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
579 * @param[in] DATA_TYPE The data type of the vectors
580 * @param[in] BASENAME The basename of the variables
581 * @param[in] PTR The base pointer
582 * @param[in] OFFSET The offset within a row
583 * @param[in] STRIDE_Y The stride value in y-axis direction
584 * @param[in] Z The offset in z-axis direction
585 * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
586 * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0.
587 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]588#define LOAD_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_N0, \
589 PARTIAL_COND_X) \
590 if (!(PARTIAL_COND_X)) \
591 { \
592 LOAD_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \
593 } \
594 else \
595 { \
596 LOAD_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]597 }
598/** Load a block that can only be partial in y but not x.
599 *
600 * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty.
601 *
602 * The data to store is expected to have consecutive names for each row.
603 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
604 * The Z offset is expected to have consecutive names.
605 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
606 *
607 * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16
608 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
609 * @param[in] DATA_TYPE The data type of the vectors
610 * @param[in] BASENAME The basename of the variables
611 * @param[in] PTR The base pointer
612 * @param[in] OFFSET The offset within a row
613 * @param[in] STRIDE_Y The stride value in y-axis direction
614 * @param[in] Z The offset in z-axis direction
615 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
616 * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
617 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]618#define LOAD_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, \
619 PARTIAL_COND_Y) \
620 if (!(PARTIAL_COND_Y)) \
621 { \
622 LOAD_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \
623 } \
624 else \
625 { \
626 LOAD_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]627 }
628/** @} */ // end of group LOAD_BLOCK_PARTIAL
629/** Boundary-aware GeMM block load
630 * @name LOAD_BLOCK_BOUNDARY_AWARE
631 * This macro assumes the following schemes to achieve boundary-awareness:
632 * - Overlapping load in Y axis from lhs tensor. This implies lhs has no padding along y dim.
633 * - Non-Overlapping(normal) load from rhs tensor. This imples rhs can have paddings.
634 * - Overlapping load in Y axis from bias tensor. This implies rhs has no padding along y dim.
635 * The macro then ensures that the src tensor can be loaded without any paddings in both x and y dim.
636 *
637 * In the y dimension, we place the partial blocks **at the beginning** while in the x dimension, we place the partial
638 * blocks **at the end**.
639 * Say, the src tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/
640 * "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters:
641 *
642 * *--x--> x == 0 x == 1
643 * | |<------------------------------N-------------------------->|
644 * y |<--------------N0------------->|<----PARTIAL_STORE_N0----->|
645 * | -------------#############################################################
646 * * | | |...............................|...........................|
647 * y == 0 | PAR_..._M0 |......Boundary block in y......|.Boundary block in x and y.|
648 * | | |...............................|...........................|
649 * M --#############################################################
650 * | | | |...........................|
651 * y == 1 | M0 | Non-boundary block |....Boundary block in x....|
652 * | | | |...........................|
653 * |------------#############################################################
654 *
655 * Then @p PARTIAL_STORE_M0 = M % M0 and @p PARTIAL_STORE_N0 = N % N0
656 *
657 * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty.
658 *
659 * It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension,
660 * and select corresponding load methods such that the boundary detection logic is only added when needed.
661 *
662 * The data to load is expected to have consecutive names for each row.
663 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
664 * The Z offset is expected to have consecutive names.
665 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
666 *
667 * The macro will result in a declaration of @p M0 vectors of size @p N0 with data
668 * type @p DATA_TYPE containing values partially loaded from the specified
669 * address in memory. The remaining (N0 - PARTIAL_STORE_N0) elements will be
670 * filled with zeros.
671 *
672 * @param[in] M0 The number of rows to load, for non-partial blocks. Supported: 1-16
673 * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
674 * @param[in] DATA_TYPE The data type of the vectors
675 * @param[in] BASENAME The basename of the variables
676 * @param[in] PTR The base pointer
677 * @param[in] OFFSET The offset within a row
678 * @param[in] STRIDE_Y The stride value in y-axis direction
679 * @param[in] Z The offset in z-axis direction
680 * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0)
681 * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported: [0, @p N0)
682 * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial load Y. True to use PARTIAL_STORE_M0 rather than M0.
683 * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0.
684 * @{
685 */
686#if PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
687// Case1: No partial blocks in either x or y
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]688#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, \
689 PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]690 LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
691
692#elif PARTIAL_STORE_M0 > 0 && PARTIAL_STORE_N0 == 0
693// Case2: Partial blocks in y
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]694#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, \
695 PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
696 REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, N0), BASENAME, 0); \
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]697 LOAD_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y)
698
699#elif PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 > 0
700// Case3: Partial blocks in x
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]701#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, \
702 PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
703 REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, N0), BASENAME, 0); \
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]704 LOAD_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X)
705
706#else // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
707// Case4: Partial blocks in both x and y
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]708#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, \
709 PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
710 REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, N0), BASENAME, 0); \
711 LOAD_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, \
712 PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X)
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]713
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]714#endif // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]715/** @} */ // end of group LOAD_BLOCK_BOUNDARY_AWARE
Giorgio Arenabde2f352021-09-07 14:15:28 +0100[diff] [blame]716
Gian Marco Iodicee3a849a2020-06-10 17:59:30 +0100[diff] [blame]717/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
718 * @name LOAD_TEXTURE2D_ROW_n
719 *
720 * @param[in] N0 The number of pixels to read
721 * @param[in] DATA_TYPE The data type of variables
722 * @param[in] BASENAME The basename of the destination variables for the loaded rows
723 * @param[in] IMG The 2D OpenCL image object
724 * @param[in] X_COORD The x coordinate for the top-left pixel
725 * @param[in] Y_COORD The y coordinate for the top-left pixel
726 * @param[in] X_STEP_ROW The incremental step row for the x coordinate (in pixels)
727 * @param[in] Y_STEP_ROW The incremental step row for the y coordinate (in pixels)
728 * @{
729 */
730#define LOAD_TEXTURE2D_ROW_1(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
731 BASENAME##0 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 0 * X_STEP_ROW), (Y_COORD + 0 * Y_STEP_ROW))
732
733#define LOAD_TEXTURE2D_ROW_2(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
734 LOAD_TEXTURE2D_ROW_1(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
735 BASENAME##1 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 1 * X_STEP_ROW), (Y_COORD + 1 * Y_STEP_ROW))
736
737#define LOAD_TEXTURE2D_ROW_3(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
738 LOAD_TEXTURE2D_ROW_2(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
739 BASENAME##2 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 2 * X_STEP_ROW), (Y_COORD + 2 * Y_STEP_ROW))
740
741#define LOAD_TEXTURE2D_ROW_4(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
742 LOAD_TEXTURE2D_ROW_3(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
743 BASENAME##3 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 3 * X_STEP_ROW), (Y_COORD + 3 * Y_STEP_ROW))
744
745#define LOAD_TEXTURE2D_ROW_5(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
746 LOAD_TEXTURE2D_ROW_4(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
747 BASENAME##4 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 4 * X_STEP_ROW), (Y_COORD + 4 * Y_STEP_ROW))
748
749#define LOAD_TEXTURE2D_ROW_6(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
750 LOAD_TEXTURE2D_ROW_5(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
751 BASENAME##5 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 5 * X_STEP_ROW), (Y_COORD + 5 * Y_STEP_ROW))
752
753#define LOAD_TEXTURE2D_ROW_7(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
754 LOAD_TEXTURE2D_ROW_6(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
755 BASENAME##6 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 6 * X_STEP_ROW), (Y_COORD + 6 * Y_STEP_ROW))
756
757#define LOAD_TEXTURE2D_ROW_8(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
758 LOAD_TEXTURE2D_ROW_7(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
759 BASENAME##7 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 7 * X_STEP_ROW), (Y_COORD + 7 * Y_STEP_ROW))
760
761#define LOAD_TEXTURE2D_ROW_9(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
762 LOAD_TEXTURE2D_ROW_8(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
763 BASENAME##8 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 8 * X_STEP_ROW), (Y_COORD + 8 * Y_STEP_ROW))
764
765#define LOAD_TEXTURE2D_ROW_10(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
766 LOAD_TEXTURE2D_ROW_9(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
767 BASENAME##9 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 9 * X_STEP_ROW), (Y_COORD + 9 * Y_STEP_ROW))
768
769#define LOAD_TEXTURE2D_ROW_11(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
770 LOAD_TEXTURE2D_ROW_10(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
771 BASENAME##A = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 10 * X_STEP_ROW), (Y_COORD + 10 * Y_STEP_ROW))
772
773#define LOAD_TEXTURE2D_ROW_12(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
774 LOAD_TEXTURE2D_ROW_11(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
775 BASENAME##B = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 11 * X_STEP_ROW), (Y_COORD + 11 * Y_STEP_ROW))
776
777#define LOAD_TEXTURE2D_ROW_13(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
778 LOAD_TEXTURE2D_ROW_12(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
779 BASENAME##C = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 12 * X_STEP_ROW), (Y_COORD + 12 * Y_STEP_ROW))
780
781#define LOAD_TEXTURE2D_ROW_14(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
782 LOAD_TEXTURE2D_ROW_13(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
783 BASENAME##D = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 13 * X_STEP_ROW), (Y_COORD + 13 * Y_STEP_ROW))
784
785#define LOAD_TEXTURE2D_ROW_15(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
786 LOAD_TEXTURE2D_ROW_14(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
787 BASENAME##E = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 14 * X_STEP_ROW), (Y_COORD + 14 * Y_STEP_ROW))
788
789#define LOAD_TEXTURE2D_ROW_16(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
790 LOAD_TEXTURE2D_ROW_15(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
791 BASENAME##F = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 15 * X_STEP_ROW), (Y_COORD + 15 * Y_STEP_ROW))
792/** @} */ // end of group LOAD_TEXTURE2D_ROW_n
793
794/** Load a 2D texture in unit of pixel. A pixel is made of 4 floating point values
795 * @name LOAD_TEXTURE2D
796 *
797 * Supported cases are M0=1,2,3,...,16 and N0=1
798 * The data to load is expected to have consecutive names for each row.
799 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
800 *
801 * @param[in] M0 The number of consecutive rows
802 * @param[in] N0 The number of consecutive pixels. Only 1, 2 and 4 are supported
803 * @param[in] DATA_TYPE The data type of the target
804 * @param[in] BASENAME The basename of the result variables
805 * @param[in] IMG The 2D OpenCL image object
806 * @param[in] X_COORD The x coordinate for the top-left pixel
807 * @param[in] Y_COORD The y coordinate for the top-left pixel
808 * @param[in] X_STEP_ROW The incremental step row for the x coordinate (in pixels)
809 * @param[in] Y_STEP_ROW The incremental step row for the y coordinate (in pixels)
810 * @{
811 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]812#define LOAD_TEXTURE2D_STR(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
813 LOAD_TEXTURE2D_ROW_##M0(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW)
814#define LOAD_TEXTURE2D(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
815 LOAD_TEXTURE2D_STR(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW)
Gian Marco Iodicee3a849a2020-06-10 17:59:30 +0100[diff] [blame]816/** @} */ // end of group LOAD_TEXTURE2D
817
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]818/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1) passing the Y index for each row to be loaded.
819 * @name LOAD_ROW_INDIRECT_n
820 *
821 * @param[in] N0 The number of columns to load
822 * @param[in] DATA_TYPE The data type of variables
823 * @param[in] BASENAME The basename of the destination variables for the loaded rows
824 * @param[in] PTR The base pointer
825 * @param[in] OFFSET The offset within a row
826 * @param[in] STRIDE_Y The stride value in y-axis direction
827 * @param[in] Y The y-axis offset vector
828 * @param[in] Y_MASK The y-axis mask vector. If 0, forces BASENAMEn to 0
829 * @{
830 */
831#define LOAD_ROW_INDIRECT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
832 VEC_DATA_TYPE(DATA_TYPE, N0) \
833 BASENAME##0; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]834 if (Y_MASK##0 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]835 BASENAME##0 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##0 * STRIDE_Y)); \
836 else \
837 BASENAME##0 = 0;
838
839#define LOAD_ROW_INDIRECT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
840 LOAD_ROW_INDIRECT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
841 VEC_DATA_TYPE(DATA_TYPE, N0) \
842 BASENAME##1; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]843 if (Y_MASK##1 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]844 BASENAME##1 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##1 * STRIDE_Y)); \
845 else \
846 BASENAME##1 = 0;
847
848#define LOAD_ROW_INDIRECT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
849 LOAD_ROW_INDIRECT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
850 VEC_DATA_TYPE(DATA_TYPE, N0) \
851 BASENAME##2; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]852 if (Y_MASK##2 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]853 BASENAME##2 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##2 * STRIDE_Y)); \
854 else \
855 BASENAME##2 = 0;
856
857#define LOAD_ROW_INDIRECT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
858 LOAD_ROW_INDIRECT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
859 VEC_DATA_TYPE(DATA_TYPE, N0) \
860 BASENAME##3; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]861 if (Y_MASK##3 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]862 BASENAME##3 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##3 * STRIDE_Y)); \
863 else \
864 BASENAME##3 = 0;
865
866#define LOAD_ROW_INDIRECT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
867 LOAD_ROW_INDIRECT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
868 VEC_DATA_TYPE(DATA_TYPE, N0) \
869 BASENAME##4; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]870 if (Y_MASK##4 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]871 BASENAME##4 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##4 * STRIDE_Y)); \
872 else \
873 BASENAME##4 = 0;
874
875#define LOAD_ROW_INDIRECT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
876 LOAD_ROW_INDIRECT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
877 VEC_DATA_TYPE(DATA_TYPE, N0) \
878 BASENAME##5; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]879 if (Y_MASK##5 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]880 BASENAME##5 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##5 * STRIDE_Y)); \
881 else \
882 BASENAME##5 = 0;
883
884#define LOAD_ROW_INDIRECT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
885 LOAD_ROW_INDIRECT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
886 VEC_DATA_TYPE(DATA_TYPE, N0) \
887 BASENAME##6; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]888 if (Y_MASK##6 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]889 BASENAME##6 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##6 * STRIDE_Y)); \
890 else \
891 BASENAME##6 = 0;
892
893#define LOAD_ROW_INDIRECT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
894 LOAD_ROW_INDIRECT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
895 VEC_DATA_TYPE(DATA_TYPE, N0) \
896 BASENAME##7; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]897 if (Y_MASK##7 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]898 BASENAME##7 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##7 * STRIDE_Y)); \
899 else \
900 BASENAME##7 = 0;
901
902#define LOAD_ROW_INDIRECT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
903 LOAD_ROW_INDIRECT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
904 VEC_DATA_TYPE(DATA_TYPE, N0) \
905 BASENAME##8; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]906 if (Y_MASK##8 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]907 BASENAME##8 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##8 * STRIDE_Y)); \
908 else \
909 BASENAME##8 = 0;
910
911#define LOAD_ROW_INDIRECT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
912 LOAD_ROW_INDIRECT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
913 VEC_DATA_TYPE(DATA_TYPE, N0) \
914 BASENAME##9; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]915 if (Y_MASK##9 != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]916 BASENAME##9 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##9 * STRIDE_Y)); \
917 else \
918 BASENAME##9 = 0;
919
920#define LOAD_ROW_INDIRECT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
921 LOAD_ROW_INDIRECT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
922 VEC_DATA_TYPE(DATA_TYPE, N0) \
923 BASENAME##A; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]924 if (Y_MASK##A != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]925 BASENAME##A = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##A * STRIDE_Y)); \
926 else \
927 BASENAME##A = 0;
928
929#define LOAD_ROW_INDIRECT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
930 LOAD_ROW_INDIRECT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
931 VEC_DATA_TYPE(DATA_TYPE, N0) \
932 BASENAME##B; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]933 if (Y_MASK##B != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]934 BASENAME##B = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##B * STRIDE_Y)); \
935 else \
936 BASENAME##B = 0;
937
938#define LOAD_ROW_INDIRECT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
939 LOAD_ROW_INDIRECT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
940 VEC_DATA_TYPE(DATA_TYPE, N0) \
941 BASENAME##C; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]942 if (Y_MASK##C != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]943 BASENAME##C = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##C * STRIDE_Y)); \
944 else \
945 BASENAME##C = 0;
946
947#define LOAD_ROW_INDIRECT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
948 LOAD_ROW_INDIRECT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
949 VEC_DATA_TYPE(DATA_TYPE, N0) \
950 BASENAME##D; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]951 if (Y_MASK##D != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]952 BASENAME##D = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##D * STRIDE_Y)); \
953 else \
954 BASENAME##D = 0;
955
956#define LOAD_ROW_INDIRECT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
957 LOAD_ROW_INDIRECT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
958 VEC_DATA_TYPE(DATA_TYPE, N0) \
959 BASENAME##E; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]960 if (Y_MASK##E != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]961 BASENAME##E = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##E * STRIDE_Y)); \
962 else \
963 BASENAME##E = 0;
964
965#define LOAD_ROW_INDIRECT_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
966 LOAD_ROW_INDIRECT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
967 VEC_DATA_TYPE(DATA_TYPE, N0) \
968 BASENAME##F; \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]969 if (Y_MASK##F != 0) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]970 BASENAME##F = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##F * STRIDE_Y)); \
971 else \
972 BASENAME##F = 0;
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]973/** @} */ // end of group LOAD_ROW_INDIRECT_n
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]974
975/** Load blocks (consecutive rows and columns) with Y offset.
976 * @name LOAD_BLOCK_INDIRECT
977 *
978 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
979 * The data to load is expected to have consecutive names for each row.
980 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
981 * The Z offset is expected to have consecutive names.
982 * E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
983 *
984 * @param[in] M0 The number of consecutive rows
985 * @param[in] N0 The number of consecutive columns
986 * @param[in] DATA_TYPE The data type of the target
987 * @param[in] BASENAME The basename of the result variables
988 * @param[in] PTR The base pointer for the data
989 * @param[in] OFFSET The offset within a row
990 * @param[in] STRIDE_Y The stride in y-axis direction
991 * @param[in] Y The y-axis offset vector
992 * @param[in] Y_MASK The y-axis mask vector. If 0, forces BASENAMEn to 0
993 * @{
994 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]995#define LOAD_BLOCK_INDIRECT_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
996 LOAD_ROW_INDIRECT_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK)
997#define LOAD_BLOCK_INDIRECT(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
998 LOAD_BLOCK_INDIRECT_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK)
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]999/** @} */ // end of group LOAD_BLOCK_INDIRECT
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]1000
Michele Di Giorgiob54ba282020-01-14 15:31:55 +0000[diff] [blame]1001/** Loads the elements from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
1002 * @name LOAD_ELEMENT_n
1003 *
1004 * @param[in] N0 The number of rows to load
1005 * @param[in] DATA_TYPE The data type of variables
1006 * @param[in] BASENAME The basename of the destination variables for the loaded rows
1007 * @param[in] PTR The base pointer
1008 * @param[in] OFFSET The offset within a row
1009 * @param[in] STRIDE_Y The stride value in y-axis direction
1010 * @{
1011 */
1012#define LOAD_ELEMENT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1013 VEC_DATA_TYPE(DATA_TYPE, N0) \
1014 BASENAME##0 = *((__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y));
1015
1016#define LOAD_ELEMENT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1017 LOAD_ELEMENT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1018 VEC_DATA_TYPE(DATA_TYPE, N0) \
1019 BASENAME##1 = *((__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y));
1020
1021#define LOAD_ELEMENT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1022 LOAD_ELEMENT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1023 VEC_DATA_TYPE(DATA_TYPE, N0) \
1024 BASENAME##2 = *((__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y));
1025
1026#define LOAD_ELEMENT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1027 LOAD_ELEMENT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1028 VEC_DATA_TYPE(DATA_TYPE, N0) \
1029 BASENAME##3 = *((__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y));
1030
1031#define LOAD_ELEMENT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1032 LOAD_ELEMENT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1033 VEC_DATA_TYPE(DATA_TYPE, N0) \
1034 BASENAME##4 = *((__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y));
1035
1036#define LOAD_ELEMENT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1037 LOAD_ELEMENT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1038 VEC_DATA_TYPE(DATA_TYPE, N0) \
1039 BASENAME##5 = *((__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y));
1040
1041#define LOAD_ELEMENT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1042 LOAD_ELEMENT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1043 VEC_DATA_TYPE(DATA_TYPE, N0) \
1044 BASENAME##6 = *((__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y));
1045
1046#define LOAD_ELEMENT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1047 LOAD_ELEMENT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1048 VEC_DATA_TYPE(DATA_TYPE, N0) \
1049 BASENAME##7 = *((__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y));
1050
1051#define LOAD_ELEMENT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1052 LOAD_ELEMENT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1053 VEC_DATA_TYPE(DATA_TYPE, N0) \
1054 BASENAME##8 = *((__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y));
1055
1056#define LOAD_ELEMENT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1057 LOAD_ELEMENT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1058 VEC_DATA_TYPE(DATA_TYPE, N0) \
1059 BASENAME##9 = *((__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y));
1060
1061#define LOAD_ELEMENT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1062 LOAD_ELEMENT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1063 VEC_DATA_TYPE(DATA_TYPE, N0) \
1064 BASENAME##A = *((__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y));
1065
1066#define LOAD_ELEMENT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1067 LOAD_ELEMENT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1068 VEC_DATA_TYPE(DATA_TYPE, N0) \
1069 BASENAME##B = *((__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y));
1070
1071#define LOAD_ELEMENT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1072 LOAD_ELEMENT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1073 VEC_DATA_TYPE(DATA_TYPE, N0) \
1074 BASENAME##C = *((__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y));
1075
1076#define LOAD_ELEMENT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1077 LOAD_ELEMENT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1078 VEC_DATA_TYPE(DATA_TYPE, N0) \
1079 BASENAME##D = *((__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y));
1080
1081#define LOAD_ELEMENT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1082 LOAD_ELEMENT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1083 VEC_DATA_TYPE(DATA_TYPE, N0) \
1084 BASENAME##E = *((__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y));
1085
1086#define LOAD_ELEMENT_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1087 LOAD_ELEMENT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1088 VEC_DATA_TYPE(DATA_TYPE, N0) \
1089 BASENAME##F = *((__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y));
1090
1091/** @}*/ // end of group LOAD_ELEMENT_n
1092
1093/** Load Scalar as Vector (consecutive elements).
1094 * @name LOAD_SCALAR_AS_VECTOR
1095 *
1096 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
1097 * The data to load is expected to have consecutive names for each row.
1098 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
1099 *
1100 * @param[in] M0 The number of consecutive rows
1101 * @param[in] N0 The number of consecutive columns
1102 * @param[in] DATA_TYPE The data type of the target
1103 * @param[in] BASENAME The basename of the result variables
1104 * @param[in] PTR The base pointer for the data
1105 * @param[in] OFFSET The offset within a row
1106 * @param[in] STRIDE_Y The stride in y-axis direction
1107 * @{
1108 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1109#define LOAD_SCALAR_AS_VECTOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1110 LOAD_ELEMENT_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y)
1111#define LOAD_SCALAR_AS_VECTOR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
1112 LOAD_SCALAR_AS_VECTOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y)
Michele Di Giorgiob54ba282020-01-14 15:31:55 +0000[diff] [blame]1113/** @} */ // end of group LOAD_SCALAR_AS_VECTOR
1114
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1115/** Basic macros to calculate Z offset values from Z0 to Zn-1
1116 * @name CALCULATE_Z_OFFSET_n
1117 *
1118 * @param[in] M0 The number of offset values to calculate
1119 * @param[in] DATA_TYPE The data type of the results
1120 * @param[in] Z The basename of the result variables
1121 * @param[in] Y The work-itme ID of y-axis
1122 * @param[in] HEIGHT_GEMM3D The height of GEMM3D
1123 * @param[in] DEPTH_GEMM3D The depth of GEMM3D
1124 * @param[in] CROSS_PLANE_PAD The padding required for plane changes accross the z-dimension
1125 * @param[in] STRIDE_Y The stride value in y-axis direction
1126 *
1127 * @{
1128 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1129#define CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]1130 Z##0 = (0 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1131 Z##0 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##0); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1132 Z##0 *= (CROSS_PLANE_PAD * STRIDE_Y);
1133
1134#define CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
1135 CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]1136 Z##1 = (1 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1137 Z##1 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##1); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1138 Z##1 *= (CROSS_PLANE_PAD * STRIDE_Y);
1139
1140#define CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
1141 CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]1142 Z##2 = (2 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1143 Z##2 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##2); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1144 Z##2 *= (CROSS_PLANE_PAD * STRIDE_Y);
1145
1146#define CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
1147 CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]1148 Z##3 = (3 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1149 Z##3 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##3); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1150 Z##3 *= (CROSS_PLANE_PAD * STRIDE_Y);
1151
1152#define CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
1153 CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]1154 Z##4 = (4 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1155 Z##4 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##4); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1156 Z##4 *= (CROSS_PLANE_PAD * STRIDE_Y);
1157
1158#define CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
1159 CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]1160 Z##5 = (5 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1161 Z##5 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##5); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1162 Z##5 *= (CROSS_PLANE_PAD * STRIDE_Y);
1163
1164#define CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
1165 CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]1166 Z##6 = (6 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1167 Z##6 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##6); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1168 Z##6 *= (CROSS_PLANE_PAD * STRIDE_Y);
1169
1170#define CALCULATE_Z_OFFSET_8(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
1171 CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]1172 Z##7 = (7 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1173 Z##7 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##7); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1174 Z##7 *= (CROSS_PLANE_PAD * STRIDE_Y);
1175
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1176/** @} */ // end of group CALCULATE_Z_OFFSET_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1177
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1178/** Calculate Z offset values from Z0 to Zn-1
1179 * @name CALCULATE_Z_OFFSET
1180 *
1181 * The Z offsets are expected to have consecutive names.
1182 * E.g., for M0=3 and Z=zin, the expected names of Z offsets are zin1, zin2, zin3.
1183 * Note that, CROSS_PLANE_PAD (cross plain padding) is required to take into account
1184 * the possible cross plane paddings in case of the plance changes across the z-dimension.
1185 *
1186 * <!--
1187 * | |
1188 * | plane0 |
1189 * | |
1190 * |__________________|
1191 * |******************|
1192 * | cross_plane_pad |
1193 * |******************|
1194 * | |
1195 * | plane1 |
1196 * | |
1197 * |__________________|
1198 * -->
1199 *
1200 * @param[in] M0 The number of offset values to calculate
1201 * @param[in] DATA_TYPE The data type of the results
1202 * @param[in] Z The basename of the result variables
1203 * @param[in] Y The work-itme ID of y-axis
1204 * @param[in] HEIGHT_GEMM3D The height of GEMM3D
1205 * @param[in] DEPTH_GEMM3D The depth of GEMM3D
1206 * @param[in] CROSS_PLANE_PAD The padding required for plane changes accross the z-dimension
1207 * @param[in] STRIDE_Y The stride value in y-axis direction
1208 * @{
1209 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1210#define CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
1211 CALCULATE_Z_OFFSET_##M0(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
1212#define CALCULATE_Z_OFFSET(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
1213 CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1214/** @} */ // end of group CALCULATE_Z_OFFSET
1215
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1216/** Scale the rows in the given variables (BASENAME0 to BASENAMEn-1)
1217 * @name SCALE_ROW_n
1218 *
1219 * @param[in] DATA_TYPE The data type of the variables
1220 * @param[in] BASENAME The basename of the variables
1221 * @param[in] SCALE The scale factor
1222 * @{
1223 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1224#define SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) BASENAME##0 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1225
1226#define SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
1227 SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1228 BASENAME##1 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1229
1230#define SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
1231 SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1232 BASENAME##2 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1233
1234#define SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
1235 SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1236 BASENAME##3 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1237
1238#define SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
1239 SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1240 BASENAME##4 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1241
1242#define SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
1243 SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1244 BASENAME##5 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1245
1246#define SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
1247 SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1248 BASENAME##6 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1249
1250#define SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
1251 SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1252 BASENAME##7 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1253
1254#define SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
1255 SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1256 BASENAME##8 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1257
1258#define SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
1259 SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1260 BASENAME##9 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1261
1262#define SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
1263 SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1264 BASENAME##A *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1265
1266#define SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
1267 SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1268 BASENAME##B *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1269
1270#define SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
1271 SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1272 BASENAME##C *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1273
1274#define SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
1275 SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1276 BASENAME##D *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1277
1278#define SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
1279 SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1280 BASENAME##E *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1281
1282#define SCALE_ROW_16(DATA_TYPE, BASENAME, SCALE) \
1283 SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1284 BASENAME##F *= (DATA_TYPE)SCALE;
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1285/** @} */ // end of group SCALE_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1286
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1287/** Scale elements stored in a block (BASENAME)
1288 * @name SCALE_BLOCK
1289 *
1290 * Supported cases are N=1,2,3,...,16
1291 *
1292 * @param[in] N The number of rows in the block
1293 * @param[in] DATA_TYPE The data type of the block
1294 * @param[in] BASENAME The basename of the block
1295 * @param[in] SCALE The scale factor
1296 * @{
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1297 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1298#define SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE) SCALE_ROW_##N(DATA_TYPE, BASENAME, SCALE)
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1299#define SCALE_BLOCK(N, DATA_TYPE, BASENAME, SCALE) SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1300/** @} */ // end of group SCALE_BLOCK
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1301
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1302/** Create a new vector containing the values at the given index for a set of given vectors
1303 * @name COLUMN_VECTORn
1304 *
1305 * @param[in] IDX_COL The index value
1306 * @param[in] BASENAME The basename of the destination vectors
1307 * @param[in] X The basename of the source vectors
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1308 * @param[in] TYPE The data type of the destination vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1309 * @{
1310 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1311#define COLUMN_VECTOR1(IDX_COL, BASENAME, X, TYPE) TYPE BASENAME##IDX_COL = (TYPE)((X##0).s##IDX_COL);
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1312#define COLUMN_VECTOR2(IDX_COL, BASENAME, X, TYPE) \
1313 VEC_DATA_TYPE(TYPE, 2) \
1314 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 2))((X##0).s##IDX_COL, (X##1).s##IDX_COL);
1315#define COLUMN_VECTOR3(IDX_COL, BASENAME, X, TYPE) \
1316 VEC_DATA_TYPE(TYPE, 3) \
1317 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 3))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL);
1318#define COLUMN_VECTOR4(IDX_COL, BASENAME, X, TYPE) \
1319 VEC_DATA_TYPE(TYPE, 4) \
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1320 BASENAME##IDX_COL = \
1321 (VEC_DATA_TYPE(TYPE, 4))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL);
1322#define COLUMN_VECTOR8(IDX_COL, BASENAME, X, TYPE) \
1323 VEC_DATA_TYPE(TYPE, 8) \
1324 BASENAME##IDX_COL = \
1325 (VEC_DATA_TYPE(TYPE, 8))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL, \
1326 (X##4).s##IDX_COL, (X##5).s##IDX_COL, (X##6).s##IDX_COL, (X##7).s##IDX_COL);
1327#define COLUMN_VECTOR16(IDX_COL, BASENAME, X, TYPE) \
1328 VEC_DATA_TYPE(TYPE, 16) \
1329 BASENAME##IDX_COL = \
1330 (VEC_DATA_TYPE(TYPE, 16))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL, \
1331 (X##4).s##IDX_COL, (X##5).s##IDX_COL, (X##6).s##IDX_COL, (X##7).s##IDX_COL, \
1332 (X##8).s##IDX_COL, (X##9).s##IDX_COL, (X##A).s##IDX_COL, (X##B).s##IDX_COL, \
1333 (X##C).s##IDX_COL, (X##D).s##IDX_COL, (X##E).s##IDX_COL, (X##F).s##IDX_COL);
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1334/** @} */ // end of group COLUMN_VECTORn
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1335
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1336/** Create a new vector containing the values at the given index. Utility macros for transposing a colum-vector
1337 * @name COLUMN_VECTOR_SCALARn
1338 *
1339 * @param[in] IDX_COL The index value
1340 * @param[in] BASENAME The basename of the destination vectors
1341 * @param[in] X The basename of the source vectors
1342 * @param[in] TYPE The data type of the destination vectors
1343 * @{
1344 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1345#define COLUMN_VECTOR_SCALAR1(IDX_COL, BASENAME, X, TYPE) TYPE BASENAME##IDX_COL = (TYPE)((X##0));
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1346#define COLUMN_VECTOR_SCALAR2(IDX_COL, BASENAME, X, TYPE) \
1347 VEC_DATA_TYPE(TYPE, 2) \
1348 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 2))((X##0), (X##1));
1349#define COLUMN_VECTOR_SCALAR3(IDX_COL, BASENAME, X, TYPE) \
1350 VEC_DATA_TYPE(TYPE, 3) \
1351 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 3))((X##0), (X##1), (X##2));
1352#define COLUMN_VECTOR_SCALAR4(IDX_COL, BASENAME, X, TYPE) \
1353 VEC_DATA_TYPE(TYPE, 4) \
1354 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 4))((X##0), (X##1), (X##2), (X##3));
1355#define COLUMN_VECTOR_SCALAR8(IDX_COL, BASENAME, X, TYPE) \
1356 VEC_DATA_TYPE(TYPE, 8) \
1357 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 8))((X##0), (X##1), (X##2), (X##3), (X##4), (X##5), (X##6), (X##7));
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1358#define COLUMN_VECTOR_SCALAR16(IDX_COL, BASENAME, X, TYPE) \
1359 VEC_DATA_TYPE(TYPE, 16) \
1360 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 16))((X##0), (X##1), (X##2), (X##3), (X##4), (X##5), (X##6), (X##7), \
1361 (X##8), (X##9), (X##A), (X##B), (X##C), (X##D), (X##E), (X##F));
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]1362/** @} */ // end of group COLUMN_VECTOR_SCALARn
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1363
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1364/** Create transposed vectors of the given vectors
1365 * @name TRANSPOSE_K0Xn
1366 *
1367 * @param[in] K0 The size of the source vectors
1368 * @param[in] BASENAME The basename of transposed vectors
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1369 * @param[in] BS The basename of source vectors for transposition
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1370 * @param[in] TYPE The data type of the transposed vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1371 * @{
1372 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1373#define TRANSPOSE_K0X1(K0, BASENAME, BS, TYPE) COLUMN_VECTOR_SCALAR(K0, 0, BASENAME, BS, TYPE);
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1374#define TRANSPOSE_K0X2(K0, BASENAME, BS, TYPE) \
1375 COLUMN_VECTOR(K0, 0, BASENAME, BS, TYPE); \
1376 COLUMN_VECTOR(K0, 1, BASENAME, BS, TYPE);
1377#define TRANSPOSE_K0X3(K0, BASENAME, BS, TYPE) \
1378 TRANSPOSE_K0X2(K0, BASENAME, BS, TYPE); \
1379 COLUMN_VECTOR(K0, 2, BASENAME, BS, TYPE);
1380#define TRANSPOSE_K0X4(K0, BASENAME, BS, TYPE) \
1381 TRANSPOSE_K0X3(K0, BASENAME, BS, TYPE); \
1382 COLUMN_VECTOR(K0, 3, BASENAME, BS, TYPE);
1383#define TRANSPOSE_K0X8(K0, BASENAME, BS, TYPE) \
1384 TRANSPOSE_K0X4(K0, BASENAME, BS, TYPE); \
1385 COLUMN_VECTOR(K0, 4, BASENAME, BS, TYPE); \
1386 COLUMN_VECTOR(K0, 5, BASENAME, BS, TYPE); \
1387 COLUMN_VECTOR(K0, 6, BASENAME, BS, TYPE); \
1388 COLUMN_VECTOR(K0, 7, BASENAME, BS, TYPE);
1389#define TRANSPOSE_K0X16(K0, BASENAME, BS, TYPE) \
1390 TRANSPOSE_K0X8(K0, BASENAME, BS, TYPE); \
1391 COLUMN_VECTOR(K0, 8, BASENAME, BS, TYPE); \
1392 COLUMN_VECTOR(K0, 9, BASENAME, BS, TYPE); \
1393 COLUMN_VECTOR(K0, A, BASENAME, BS, TYPE); \
1394 COLUMN_VECTOR(K0, B, BASENAME, BS, TYPE); \
1395 COLUMN_VECTOR(K0, C, BASENAME, BS, TYPE); \
1396 COLUMN_VECTOR(K0, D, BASENAME, BS, TYPE); \
1397 COLUMN_VECTOR(K0, E, BASENAME, BS, TYPE); \
1398 COLUMN_VECTOR(K0, F, BASENAME, BS, TYPE);
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1399
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1400/** @} */ // end of group TRANSPOSE_K0Xn
1401
1402/** Create column vectors to contain the values at the given index for a set of given vectors
1403 *
1404 * @param[in] K0 The number of source vectors
1405 * @param[in] IDX_COL The index value
1406 * @param[in] BASENAME The basename of the destination vectors
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1407 * @param[in] BS The basename of the source vectors
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1408 * @param[in] TYPE The data type of the destination vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1409 */
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1410#define COLUMN_VECTOR(K0, IDX_COL, BASENAME, BS, TYPE) \
1411 CONCAT(COLUMN_VECTOR, K0) \
1412 (IDX_COL, BASENAME, BS, TYPE);
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1413
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1414/** Create column vectors to contain the values at the given index. Utility macro for transposing a column-vector
1415 *
1416 * @param[in] K0 The number of source vectors
1417 * @param[in] IDX_COL The index value
1418 * @param[in] BASENAME The basename of the destination vectors
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1419 * @param[in] BS The basename of the source vectors
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1420 * @param[in] TYPE The data type of the destination vectors
1421 */
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1422#define COLUMN_VECTOR_SCALAR(K0, IDX_COL, BASENAME, BS, TYPE) \
1423 CONCAT(COLUMN_VECTOR_SCALAR, K0) \
1424 (IDX_COL, BASENAME, BS, TYPE);
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1425
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1426/** Create transposed vectors form the given source vectors
1427 *
1428 * @param[in] K0 The size of source vectors
1429 * @param[in] N0 The number of source vectors
1430 * @param[in] BASENAME The basename of transposed vectors
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1431 * @param[in] BS The basename of source vectors for transposition
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1432 * @param[in] TYPE The data type of the transposed vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1433 *
1434 */
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1435#define TRANSPOSE_K0XN0(K0, N0, BASENAME, BS, TYPE) \
1436 CONCAT(TRANSPOSE_K0X, N0) \
1437 (K0, BASENAME, BS, TYPE);
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1438
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1439/** Add the variables (BIAS0 to BIASn-1) to the others (BASENAME0 to BASENAMEn-1)
1440 * @name ADD_ROW_n
1441 *
1442 * @param[in] BASENAME The basename of the destination variables
1443 * @param[in] BIAS The basename of the added variables
1444 * @{
1445 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1446#define ADD_ROW_1(BASENAME, BIAS) BASENAME##0 += BIAS##0;
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1447
1448#define ADD_ROW_2(BASENAME, BIAS) \
1449 ADD_ROW_1(BASENAME, BIAS) \
1450 BASENAME##1 += BIAS##1;
1451
1452#define ADD_ROW_3(BASENAME, BIAS) \
1453 ADD_ROW_2(BASENAME, BIAS) \
1454 BASENAME##2 += BIAS##2;
1455
1456#define ADD_ROW_4(BASENAME, BIAS) \
1457 ADD_ROW_3(BASENAME, BIAS) \
1458 BASENAME##3 += BIAS##3;
1459
1460#define ADD_ROW_5(BASENAME, BIAS) \
1461 ADD_ROW_4(BASENAME, BIAS) \
1462 BASENAME##4 += BIAS##4;
1463
1464#define ADD_ROW_6(BASENAME, BIAS) \
1465 ADD_ROW_5(BASENAME, BIAS) \
1466 BASENAME##5 += BIAS##5;
1467
1468#define ADD_ROW_7(BASENAME, BIAS) \
1469 ADD_ROW_6(BASENAME, BIAS) \
1470 BASENAME##6 += BIAS##6;
1471
1472#define ADD_ROW_8(BASENAME, BIAS) \
1473 ADD_ROW_7(BASENAME, BIAS) \
1474 BASENAME##7 += BIAS##7;
1475
1476#define ADD_ROW_9(BASENAME, BIAS) \
1477 ADD_ROW_8(BASENAME, BIAS) \
1478 BASENAME##8 += BIAS##8;
1479
1480#define ADD_ROW_10(BASENAME, BIAS) \
1481 ADD_ROW_9(BASENAME, BIAS) \
1482 BASENAME##9 += BIAS##9;
1483
1484#define ADD_ROW_11(BASENAME, BIAS) \
1485 ADD_ROW_10(BASENAME, BIAS) \
1486 BASENAME##A += BIAS##A;
1487
1488#define ADD_ROW_12(BASENAME, BIAS) \
1489 ADD_ROW_11(BASENAME, BIAS) \
1490 BASENAME##B += BIAS##B;
1491
1492#define ADD_ROW_13(BASENAME, BIAS) \
1493 ADD_ROW_12(BASENAME, BIAS) \
1494 BASENAME##C += BIAS##C;
1495
1496#define ADD_ROW_14(BASENAME, BIAS) \
1497 ADD_ROW_13(BASENAME, BIAS) \
1498 BASENAME##D += BIAS##D;
1499
1500#define ADD_ROW_15(BASENAME, BIAS) \
1501 ADD_ROW_14(BASENAME, BIAS) \
1502 BASENAME##E += BIAS##E;
1503
1504#define ADD_ROW_16(BASENAME, BIAS) \
1505 ADD_ROW_15(BASENAME, BIAS) \
1506 BASENAME##F += BIAS##F;
1507
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1508/** @} */ // end of group ADD_ROW_n
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1509
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1510/** Add the block (BIAS) to another block (BASENAME)
1511 * @name ADD_BLOCK
1512 *
1513 * Supported cases are N=1,2,3,...,16
1514 *
1515 * @param[in] N The number of vectors in the block
1516 * @param[in] BASENAME The basename of the destination variables
1517 * @param[in] BIAS The basename of the added variables
1518 * @{
1519 */
1520#define ADD_BLOCK_STR(N, BASENAME, BIAS) ADD_ROW_##N(BASENAME, BIAS)
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1521#define ADD_BLOCK(N, BASENAME, BIAS) ADD_BLOCK_STR(N, BASENAME, BIAS)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1522/** @} */ // end of group ADD_BLOCK
1523
1524/** Broadcast (add single value) to the each element of the destination variables
1525 * @name ADD_ROW_BROADCAST_n
1526 *
1527 * @param[in] BASENAME The basename of the destination variables
1528 * @param[in] BIAS The variable containing the value to add
1529 * @{
1530 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1531#define ADD_ROW_BROADCAST_1(BASENAME, BIAS) BASENAME##0 += BIAS;
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1532
1533#define ADD_ROW_BROADCAST_2(BASENAME, BIAS) \
1534 ADD_ROW_BROADCAST_1(BASENAME, BIAS) \
1535 BASENAME##1 += BIAS;
1536
1537#define ADD_ROW_BROADCAST_3(BASENAME, BIAS) \
1538 ADD_ROW_BROADCAST_2(BASENAME, BIAS) \
1539 BASENAME##2 += BIAS;
1540
1541#define ADD_ROW_BROADCAST_4(BASENAME, BIAS) \
1542 ADD_ROW_BROADCAST_3(BASENAME, BIAS) \
1543 BASENAME##3 += BIAS;
1544
1545#define ADD_ROW_BROADCAST_5(BASENAME, BIAS) \
1546 ADD_ROW_BROADCAST_4(BASENAME, BIAS) \
1547 BASENAME##4 += BIAS;
1548
1549#define ADD_ROW_BROADCAST_6(BASENAME, BIAS) \
1550 ADD_ROW_BROADCAST_5(BASENAME, BIAS) \
1551 BASENAME##5 += BIAS;
1552
1553#define ADD_ROW_BROADCAST_7(BASENAME, BIAS) \
1554 ADD_ROW_BROADCAST_6(BASENAME, BIAS) \
1555 BASENAME##6 += BIAS;
1556
1557#define ADD_ROW_BROADCAST_8(BASENAME, BIAS) \
1558 ADD_ROW_BROADCAST_7(BASENAME, BIAS) \
1559 BASENAME##7 += BIAS;
1560
1561#define ADD_ROW_BROADCAST_9(BASENAME, BIAS) \
1562 ADD_ROW_BROADCAST_8(BASENAME, BIAS) \
1563 BASENAME##8 += BIAS;
1564
1565#define ADD_ROW_BROADCAST_10(BASENAME, BIAS) \
1566 ADD_ROW_BROADCAST_9(BASENAME, BIAS) \
1567 BASENAME##9 += BIAS;
1568
1569#define ADD_ROW_BROADCAST_11(BASENAME, BIAS) \
1570 ADD_ROW_BROADCAST_10(BASENAME, BIAS) \
1571 BASENAME##A += BIAS;
1572
1573#define ADD_ROW_BROADCAST_12(BASENAME, BIAS) \
1574 ADD_ROW_BROADCAST_11(BASENAME, BIAS) \
1575 BASENAME##B += BIAS;
1576
1577#define ADD_ROW_BROADCAST_13(BASENAME, BIAS) \
1578 ADD_ROW_BROADCAST_12(BASENAME, BIAS) \
1579 BASENAME##C += BIAS;
1580
1581#define ADD_ROW_BROADCAST_14(BASENAME, BIAS) \
1582 ADD_ROW_BROADCAST_13(BASENAME, BIAS) \
1583 BASENAME##D += BIAS;
1584
1585#define ADD_ROW_BROADCAST_15(BASENAME, BIAS) \
1586 ADD_ROW_BROADCAST_14(BASENAME, BIAS) \
1587 BASENAME##E += BIAS;
1588
1589#define ADD_ROW_BROADCAST_16(BASENAME, BIAS) \
1590 ADD_ROW_BROADCAST_15(BASENAME, BIAS) \
1591 BASENAME##F += BIAS;
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]1592/** @} */ // end of group ADD_ROW_BROADCAST_n
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1593
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1594/** Broadcast (add a value) to the each element of the destination block (BASENAME)
1595 * @name ADD_BLOCK_BROADCAST
1596 *
1597 * Supported cases are N=1,2,3,...,16.
1598 *
1599 * @param[in] N The number of vectors in the block
1600 * @param[in] BASENAME The basename of the destination variables
1601 * @param[in] BIAS The variable containing the value to add
1602 * @{
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1603 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1604#define ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS) ADD_ROW_BROADCAST_##N(BASENAME, BIAS)
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1605#define ADD_BLOCK_BROADCAST(N, BASENAME, BIAS) ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1606/** @} */ // end of group ADD_BLOCK_BROADCAST
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1607
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1608/** Apply activation to the given variables
1609 * @name ACTIVATION_ROW_n
1610 *
1611 * @param[in] ACTIVATION_TYPE The type of the activation
1612 * @param[in] DATA_TYPE The data type of the vectors
1613 * @param[in] BASENAME The basename of the variables
1614 * @param[in] A_VAL Additional value required by the activation
1615 * @param[in] B_VAL Additional value required by the activation
1616 * @{
1617 */
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1618#define ACTIVATION_ROW_1(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1619 BASENAME##0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##0, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1620
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1621#define ACTIVATION_ROW_2(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1622 ACTIVATION_ROW_1(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1623 BASENAME##1 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##1, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1624
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1625#define ACTIVATION_ROW_3(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1626 ACTIVATION_ROW_2(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1627 BASENAME##2 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##2, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1628
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1629#define ACTIVATION_ROW_4(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1630 ACTIVATION_ROW_3(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1631 BASENAME##3 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##3, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1632
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1633#define ACTIVATION_ROW_5(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1634 ACTIVATION_ROW_4(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1635 BASENAME##4 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##4, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1636
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1637#define ACTIVATION_ROW_6(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1638 ACTIVATION_ROW_5(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1639 BASENAME##5 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##5, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1640
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1641#define ACTIVATION_ROW_7(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1642 ACTIVATION_ROW_6(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1643 BASENAME##6 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##6, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1644
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1645#define ACTIVATION_ROW_8(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1646 ACTIVATION_ROW_7(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1647 BASENAME##7 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##7, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1648
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1649#define ACTIVATION_ROW_9(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1650 ACTIVATION_ROW_8(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1651 BASENAME##8 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##8, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1652
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1653#define ACTIVATION_ROW_10(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1654 ACTIVATION_ROW_9(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1655 BASENAME##9 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##9, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1656
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1657#define ACTIVATION_ROW_11(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1658 ACTIVATION_ROW_10(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1659 BASENAME##A = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##A, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1660
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1661#define ACTIVATION_ROW_12(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1662 ACTIVATION_ROW_11(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1663 BASENAME##B = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##B, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1664
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1665#define ACTIVATION_ROW_13(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1666 ACTIVATION_ROW_12(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1667 BASENAME##C = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##C, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1668
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1669#define ACTIVATION_ROW_14(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1670 ACTIVATION_ROW_13(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1671 BASENAME##D = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##D, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1672
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1673#define ACTIVATION_ROW_15(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1674 ACTIVATION_ROW_14(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1675 BASENAME##E = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##E, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1676
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1677#define ACTIVATION_ROW_16(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1678 ACTIVATION_ROW_15(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1679 BASENAME##F = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##F, A_VAL, B_VAL);
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1680/** @} */ // end of group ACTIVATION_ROW_n
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1681
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1682/** Apply activation to a block (BASENAME)
1683 * @name ACTIVATION_BLOCK
1684 *
1685 * Supported cases are N=1,2,3,...,16.
1686 *
1687 * @param[in] N The number of vectors in the block
1688 * @param[in] ACTIVATION_TYPE The type of the activation
1689 * @param[in] DATA_TYPE The data type of the vectors
1690 * @param[in] BASENAME The basename of the variables
1691 * @param[in] A_VAL Additional value required by the activation
1692 * @param[in] B_VAL Additional value required by the activation
1693 * @{
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1694 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1695#define ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1696 ACTIVATION_ROW_##N(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL)
1697#define ACTIVATION_BLOCK(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1698 ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1699/** @} */ // end of group ACTIVATION_BLOCK
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1700
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1701/** Apply convert_<data_type> to the given variables
1702 * @name CONVERT_ROW_n
1703 *
1704 * @param[in] N The size of the vectors
1705 * @param[in] DATA_TYPE The data type of the vectors
1706 * @param[in] BASENAME_SRC The basename of the source variables
1707 * @param[in] BASENAME_DST The basename of the destination variables
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]1708 * @{
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1709 */
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1710#define CONVERT_ROW_1(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1711 VEC_DATA_TYPE(DATA_TYPE, N) \
1712 BASENAME_DST##0 = CONVERT(BASENAME_SRC##0, VEC_DATA_TYPE(DATA_TYPE, N));
1713
1714#define CONVERT_ROW_2(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1715 CONVERT_ROW_1(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1716 VEC_DATA_TYPE(DATA_TYPE, N) \
1717 BASENAME_DST##1 = CONVERT(BASENAME_SRC##1, VEC_DATA_TYPE(DATA_TYPE, N));
1718
1719#define CONVERT_ROW_3(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1720 CONVERT_ROW_2(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1721 VEC_DATA_TYPE(DATA_TYPE, N) \
1722 BASENAME_DST##2 = CONVERT(BASENAME_SRC##2, VEC_DATA_TYPE(DATA_TYPE, N));
1723
1724#define CONVERT_ROW_4(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1725 CONVERT_ROW_3(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1726 VEC_DATA_TYPE(DATA_TYPE, N) \
1727 BASENAME_DST##3 = CONVERT(BASENAME_SRC##3, VEC_DATA_TYPE(DATA_TYPE, N));
1728
1729#define CONVERT_ROW_5(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1730 CONVERT_ROW_4(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1731 VEC_DATA_TYPE(DATA_TYPE, N) \
1732 BASENAME_DST##4 = CONVERT(BASENAME_SRC##4, VEC_DATA_TYPE(DATA_TYPE, N));
1733
1734#define CONVERT_ROW_6(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1735 CONVERT_ROW_5(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1736 VEC_DATA_TYPE(DATA_TYPE, N) \
1737 BASENAME_DST##5 = CONVERT(BASENAME_SRC##5, VEC_DATA_TYPE(DATA_TYPE, N));
1738
1739#define CONVERT_ROW_7(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1740 CONVERT_ROW_6(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1741 VEC_DATA_TYPE(DATA_TYPE, N) \
1742 BASENAME_DST##6 = CONVERT(BASENAME_SRC##6, VEC_DATA_TYPE(DATA_TYPE, N));
1743
1744#define CONVERT_ROW_8(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1745 CONVERT_ROW_7(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1746 VEC_DATA_TYPE(DATA_TYPE, N) \
1747 BASENAME_DST##7 = CONVERT(BASENAME_SRC##7, VEC_DATA_TYPE(DATA_TYPE, N));
1748
1749#define CONVERT_ROW_9(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1750 CONVERT_ROW_8(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1751 VEC_DATA_TYPE(DATA_TYPE, N) \
1752 BASENAME_DST##8 = CONVERT(BASENAME_SRC##8, VEC_DATA_TYPE(DATA_TYPE, N));
1753
1754#define CONVERT_ROW_10(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1755 CONVERT_ROW_9(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1756 VEC_DATA_TYPE(DATA_TYPE, N) \
1757 BASENAME_DST##9 = CONVERT(BASENAME_SRC##9, VEC_DATA_TYPE(DATA_TYPE, N));
1758
1759#define CONVERT_ROW_11(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1760 CONVERT_ROW_10(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1761 VEC_DATA_TYPE(DATA_TYPE, N) \
1762 BASENAME_DST##A = CONVERT(BASENAME_SRC##A, VEC_DATA_TYPE(DATA_TYPE, N));
1763
1764#define CONVERT_ROW_12(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1765 CONVERT_ROW_11(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1766 VEC_DATA_TYPE(DATA_TYPE, N) \
1767 BASENAME_DST##B = CONVERT(BASENAME_SRC##B, VEC_DATA_TYPE(DATA_TYPE, N));
1768
1769#define CONVERT_ROW_13(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1770 CONVERT_ROW_12(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1771 VEC_DATA_TYPE(DATA_TYPE, N) \
1772 BASENAME_DST##C = CONVERT(BASENAME_SRC##C, VEC_DATA_TYPE(DATA_TYPE, N));
1773
1774#define CONVERT_ROW_14(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1775 CONVERT_ROW_13(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1776 VEC_DATA_TYPE(DATA_TYPE, N) \
1777 BASENAME_DST##D = CONVERT(BASENAME_SRC##D, VEC_DATA_TYPE(DATA_TYPE, N));
1778
1779#define CONVERT_ROW_15(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1780 CONVERT_ROW_14(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1781 VEC_DATA_TYPE(DATA_TYPE, N) \
1782 BASENAME_DST##E = CONVERT(BASENAME_SRC##E, VEC_DATA_TYPE(DATA_TYPE, N));
1783
1784#define CONVERT_ROW_16(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1785 CONVERT_ROW_15(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1786 VEC_DATA_TYPE(DATA_TYPE, N) \
1787 BASENAME_DST##F = CONVERT(BASENAME_SRC##F, VEC_DATA_TYPE(DATA_TYPE, N));
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1788/** @} */ // end of group CONVERT_ROW_n
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1789
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1790/** Apply convert_<data_type> to a block (BASENAME_SRC) and save to another block (BASENAME_DST)
1791 * @name CONVERT_BLOCK
1792 *
1793 * Supported cases N=1,2,3,...,16.
1794 *
1795 * @param[in] M The number of vectors to convert
1796 * @param[in] N The size of the vectors
1797 * @param[in] DATA_TYPE The data type of the vectors
1798 * @param[in] BASENAME_SRC The basename of the source variables
1799 * @param[in] BASENAME_DST The basename of the destination variables
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]1800 * @{
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1801 */
Felix Thomasmathibalanafd38f02023-09-27 17:46:17 +0100[diff] [blame]1802#define CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1803 CONVERT_ROW_##M(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST)
1804#define CONVERT_BLOCK(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1805 CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST)
ramy.elgammal@arm.coma2561f02023-06-16 20:45:48 +0100[diff] [blame]1806/** @} */ // end of group CONVERT_BLOCK