Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / 4e53c5ab47a713ab0ce53d076e2e4cf274fec312 / . / src / core / CL / cl_kernels / gemm_helpers.h
blob: 3bbd243ff5ccc1241fd972afe1bbb26554006272 [file] [log] [blame]
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1/*
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]2 * Copyright (c) 2019-2021 Arm Limited.
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]24#include "activation_float_helpers.h"
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]25#include "helpers.h"
26
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]27/** Utility macro to access a vector with the scalar positions
28 *
29 * Supported cases are: Offset can only be of the same size of the OpenCL vector (2,3,4,8,16)
30 *
31 * @param[in] offset The offset within the vector. Offset can only be of the same size of the OpenCL vector (2,3,4,8,16)
32 * @param[in] n0 The number of consecutive columns to access. n0 + offset must be <= 16
33 * @param[in] x Vector to access
34 * @{
35 */
36#define SCALAR_ACCESS_STR(offset, n0, x) scalar_access_##offset##_##n0(x)
37#define SCALAR_ACCESS(offset, n0, x) SCALAR_ACCESS_STR(offset, n0, x)
38
39// offset == 0
40#define scalar_access_0_1(x) ((x).s0)
41#define scalar_access_0_2(x) ((x).s01)
42#define scalar_access_0_3(x) ((x).s012)
43#define scalar_access_0_4(x) ((x).s0123)
44#define scalar_access_0_8(x) ((x).s01234567)
45#define scalar_access_0_16(x) ((x).s0123456789ABCDEF)
46
47// offset == 1
48#define scalar_access_1_1(x) ((x).s1)
49#define scalar_access_1_2(x) ((x).s12)
50#define scalar_access_1_3(x) ((x).s123)
51#define scalar_access_1_4(x) ((x).s1234)
52#define scalar_access_1_8(x) ((x).s12345678)
53
54// offset == 2
55#define scalar_access_2_1(x) ((x).s2)
56#define scalar_access_2_2(x) ((x).s23)
57#define scalar_access_2_3(x) ((x).s234)
58#define scalar_access_2_4(x) ((x).s2345)
59#define scalar_access_2_8(x) ((x).s23456789)
60
61// offset == 3
62#define scalar_access_3_1(x) ((x).s3)
63#define scalar_access_3_2(x) ((x).s34)
64#define scalar_access_3_3(x) ((x).s345)
65#define scalar_access_3_4(x) ((x).s3456)
66#define scalar_access_3_8(x) ((x).s3456789A)
67
68// offset == 4
69#define scalar_access_4_1(x) ((x).s4)
70#define scalar_access_4_2(x) ((x).s45)
71#define scalar_access_4_3(x) ((x).s456)
72#define scalar_access_4_4(x) ((x).s4567)
73#define scalar_access_4_8(x) ((x).s456789AB)
74
75// offset == 8
76#define scalar_access_8_1(x) ((x).s8)
77#define scalar_access_8_2(x) ((x).s89)
78#define scalar_access_8_3(x) ((x).s89A)
79#define scalar_access_8_4(x) ((x).s89AB)
80#define scalar_access_8_8(x) ((x).s89ABCDEF)
81
82// offset == 12
83#define scalar_access_12_1(x) ((x).sC)
84#define scalar_access_12_2(x) ((x).sCD)
85#define scalar_access_12_3(x) ((x).sCDE)
86#define scalar_access_12_4(x) ((x).sCDEF)
87
88// offset == 16
89#define scalar_access_16_1(x) ((x).sF)
90
91/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1) without allocating variables.
92 * @name LOAD_TENSOR_ROW_n
93 *
94 * @param[in] N0 The number of columns to load
95 * @param[in] DATA_TYPE The data type of variables
96 * @param[in] BASENAME The basename of the destination variables for the loaded rows
97 * @param[in] PTR The base pointer
98 * @param[in] COL_OFFSET The column vector offset. COL_OFFSET + N0 must be <= 16
99 * @param[in] STRIDE_Y The stride value in y-axis direction
100 * @param[in] Z The z-axis offset vector
101 * @{
102 */
103#define LOAD_TENSOR_ROW_0(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
104 ({})
105
106#define LOAD_TENSOR_ROW_1(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
107 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##0) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
108
109#define LOAD_TENSOR_ROW_2(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
110 LOAD_TENSOR_ROW_1(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
111 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##1) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
112
113#define LOAD_TENSOR_ROW_3(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
114 LOAD_TENSOR_ROW_2(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
115 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##2) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
116
117#define LOAD_TENSOR_ROW_4(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
118 LOAD_TENSOR_ROW_3(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
119 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##3) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
120
121#define LOAD_TENSOR_ROW_5(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
122 LOAD_TENSOR_ROW_4(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
123 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##4) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
124
125#define LOAD_TENSOR_ROW_6(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
126 LOAD_TENSOR_ROW_5(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
127 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##5) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
128
129#define LOAD_TENSOR_ROW_7(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
130 LOAD_TENSOR_ROW_6(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
131 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##6) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
132
133#define LOAD_TENSOR_ROW_8(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
134 LOAD_TENSOR_ROW_7(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
135 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##7) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
136
137#define LOAD_TENSOR_ROW_9(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
138 LOAD_TENSOR_ROW_8(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
139 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##8) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
140
141#define LOAD_TENSOR_ROW_10(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
142 LOAD_TENSOR_ROW_9(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
143 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##9) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
144
145#define LOAD_TENSOR_ROW_11(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
146 LOAD_TENSOR_ROW_10(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
147 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##A) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
148
149#define LOAD_TENSOR_ROW_12(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
150 LOAD_TENSOR_ROW_11(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
151 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##B) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
152
153#define LOAD_TENSOR_ROW_13(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
154 LOAD_TENSOR_ROW_12(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
155 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##C) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
156
157#define LOAD_TENSOR_ROW_14(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
158 LOAD_TENSOR_ROW_13(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
159 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##D) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
160
161#define LOAD_TENSOR_ROW_15(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
162 LOAD_TENSOR_ROW_14(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
163 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##E) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
164
165#define LOAD_TENSOR_ROW_16(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
166 LOAD_TENSOR_ROW_15(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
167 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##F) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
168/** @}*/ // end of group LOAD_TENSOR_ROW_n
169
170/** Load tensor (consecutive rows and columns) with Z offset.
171 * @name LOAD_TENSOR
172 *
173 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
174 * The data to load is expected to have consecutive names for each row.
175 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
176 * The Z offset is expected to have consecutive names.
177 * E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
178 *
179 * @param[in] M0 The number of consecutive rows
180 * @param[in] N0 The number of consecutive columns
181 * @param[in] DATA_TYPE The data type of the target
182 * @param[in] BASENAME The basename of the result variables
183 * @param[in] PTR The base pointer for the data
184 * @param[in] COL_OFFSET The column vector offset. COL_OFFSET + N0 must be <= 16
185 * @param[in] STRIDE_Y The stride in y-axis direction
186 * @param[in] Z The z-axis offset vector
187 * @{
188 */
189#define LOAD_TENSOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) LOAD_TENSOR_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z)
190#define LOAD_TENSOR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) LOAD_TENSOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z)
191/** @} */ // end of group LOAD_TENSOR
192
193/** Load 2D tensor (consecutive rows and columns) with Z offset.
194 * @name LOAD_TENSOR_M0Xn
195 *
196 * @param[in] M0 The number of rows to load [0-16]
197 * @param[in] N0 The number of columns to load [0-16]
198 * @param[in] DATA_TYPE The data type of variables
199 * @param[in] BASENAME The basename of the destination variables for the loaded rows
200 * @param[in] PTR The base pointer
201 * @param[in] STRIDE_Y The stride value in y-axis direction
202 * @param[in] Z The z-axis offset vector
203 * @{
204 */
205#define LOAD_TENSOR_M0X0(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
206 ({})
207
208#define LOAD_TENSOR_M0X1(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
209 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
210
211#define LOAD_TENSOR_M0X2(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
212 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
213
214#define LOAD_TENSOR_M0X3(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
215 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
216
217#define LOAD_TENSOR_M0X4(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
218 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
219
220#define LOAD_TENSOR_M0X5(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
221 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
222 LOAD_TENSOR(M0, 1, DATA_TYPE, a, input_ptr + 4 * sizeof(DATA_TYPE), 4, src_stride_y, zin);
223
224#define LOAD_TENSOR_M0X6(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
225 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
226 LOAD_TENSOR(M0, 2, DATA_TYPE, a, input_ptr + 4 * sizeof(DATA_TYPE), 4, src_stride_y, zin);
227
228#define LOAD_TENSOR_M0X7(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
229 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
230 LOAD_TENSOR(M0, 3, DATA_TYPE, a, input_ptr + 4 * sizeof(DATA_TYPE), 4, src_stride_y, zin);
231
232#define LOAD_TENSOR_M0X8(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
233 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
234
235#define LOAD_TENSOR_M0X9(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
236 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr 0, src_stride_y, zin); \
237 LOAD_TENSOR(M0, 1, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
238
239#define LOAD_TENSOR_M0X10(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
240 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
241 LOAD_TENSOR(M0, 2, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
242
243#define LOAD_TENSOR_M0X11(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
244 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
245 LOAD_TENSOR(M0, 3, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
246
247#define LOAD_TENSOR_M0X12(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
248 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
249 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
250
251#define LOAD_TENSOR_M0X13(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
252 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
253 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin); \
254 LOAD_TENSOR(M0, 1, DATA_TYPE, a, input_ptr + 12 * sizeof(DATA_TYPE), 12, src_stride_y, zin);
255
256#define LOAD_TENSOR_M0X14(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
257 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr 0, src_stride_y, zin); \
258 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin); \
259 LOAD_TENSOR(M0, 2, DATA_TYPE, a, input_ptr + 12 * sizeof(DATA_TYPE), 12, src_stride_y, zin);
260
261#define LOAD_TENSOR_M0X15(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
262 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
263 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin); \
264 LOAD_TENSOR(M0, 3, DATA_TYPE, a, input_ptr + 12 * sizeof(DATA_TYPE), 12, src_stride_y, zin);
265
266#define LOAD_TENSOR_M0X16(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
267 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
268/** @}*/ // end of group LOAD_TENSOR_M0Xn
269
270/** Load 2D tensor (consecutive rows and columns) with Z offset.
271 * @name LOAD_TENSOR_M0XN0
272 *
273 * @param[in] M0 The number of consecutive rows [0-16]
274 * @param[in] N0 The number of consecutive columns [0-16]
275 * @param[in] DATA_TYPE The data type of the target
276 * @param[in] BASENAME The basename of the result variables
277 * @param[in] PTR The base pointer for the data
278 * @param[in] STRIDE_Y The stride in y-axis direction
279 * @param[in] Z The z-axis offset vector
280 * @{
281 */
282#define LOAD_TENSOR_M0XN0_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) LOAD_TENSOR_M0X##N0(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
283#define LOAD_TENSOR_M0XN0(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) LOAD_TENSOR_M0XN0_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
284
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]285/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
286 * @name LOAD_ROW_n
287 *
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]288 * @param[in] N0 The number of columns to load
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]289 * @param[in] DATA_TYPE The data type of variables
290 * @param[in] BASENAME The basename of the destination variables for the loaded rows
291 * @param[in] PTR The base pointer
292 * @param[in] OFFSET The offset within a row
293 * @param[in] STRIDE_Y The stride value in y-axis direction
294 * @param[in] Z The z-axis offset vector
295 * @{
296 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]297#define LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
298 VEC_DATA_TYPE(DATA_TYPE, N0) \
299 BASENAME##0 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y + Z##0));
300
301#define LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
302 LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
303 VEC_DATA_TYPE(DATA_TYPE, N0) \
304 BASENAME##1 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y + Z##1));
305
306#define LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
307 LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
308 VEC_DATA_TYPE(DATA_TYPE, N0) \
309 BASENAME##2 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y + Z##2));
310
311#define LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
312 LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
313 VEC_DATA_TYPE(DATA_TYPE, N0) \
314 BASENAME##3 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y + Z##3));
315
316#define LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
317 LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
318 VEC_DATA_TYPE(DATA_TYPE, N0) \
319 BASENAME##4 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y + Z##4));
320
321#define LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
322 LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
323 VEC_DATA_TYPE(DATA_TYPE, N0) \
324 BASENAME##5 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y + Z##5));
325
326#define LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
327 LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
328 VEC_DATA_TYPE(DATA_TYPE, N0) \
329 BASENAME##6 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y + Z##6));
330
331#define LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
332 LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
333 VEC_DATA_TYPE(DATA_TYPE, N0) \
334 BASENAME##7 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y + Z##7));
335
336#define LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
337 LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
338 VEC_DATA_TYPE(DATA_TYPE, N0) \
339 BASENAME##8 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y + Z##8));
340
341#define LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
342 LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
343 VEC_DATA_TYPE(DATA_TYPE, N0) \
344 BASENAME##9 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y + Z##9));
345
346#define LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
347 LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
348 VEC_DATA_TYPE(DATA_TYPE, N0) \
349 BASENAME##A = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y + Z##A));
350
351#define LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
352 LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
353 VEC_DATA_TYPE(DATA_TYPE, N0) \
354 BASENAME##B = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y + Z##B));
355
356#define LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
357 LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
358 VEC_DATA_TYPE(DATA_TYPE, N0) \
359 BASENAME##C = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y + Z##C));
360
361#define LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
362 LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
363 VEC_DATA_TYPE(DATA_TYPE, N0) \
364 BASENAME##D = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y + Z##D));
365
366#define LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
367 LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
368 VEC_DATA_TYPE(DATA_TYPE, N0) \
369 BASENAME##E = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y + Z##E));
370
371#define LOAD_ROW_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
372 LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
373 VEC_DATA_TYPE(DATA_TYPE, N0) \
374 BASENAME##F = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y + Z##F));
375
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]376/** @}*/ // end of group LOAD_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]377
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]378/** Load Blocks (consecutive rows and columns) with Z offset.
379 * @name LOAD_BLOCK
380 *
381 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
382 * The data to load is expected to have consecutive names for each row.
383 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
384 * The Z offset is expected to have consecutive names.
385 * E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
386 *
387 * @param[in] M0 The number of consecutive rows
388 * @param[in] N0 The number of consecutive columns
389 * @param[in] DATA_TYPE The data type of the target
390 * @param[in] BASENAME The basename of the result variables
391 * @param[in] PTR The base pointer for the data
392 * @param[in] OFFSET The offset within a row
393 * @param[in] STRIDE_Y The stride in y-axis direction
394 * @param[in] Z The z-axis offset vector
395 * @{
396 */
397#define LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
398#define LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
399/** @} */ // end of group LOAD_BLOCK
400
Gian Marco Iodicee3a849a2020-06-10 17:59:30 +0100[diff] [blame]401/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
402 * @name LOAD_TEXTURE2D_ROW_n
403 *
404 * @param[in] N0 The number of pixels to read
405 * @param[in] DATA_TYPE The data type of variables
406 * @param[in] BASENAME The basename of the destination variables for the loaded rows
407 * @param[in] IMG The 2D OpenCL image object
408 * @param[in] X_COORD The x coordinate for the top-left pixel
409 * @param[in] Y_COORD The y coordinate for the top-left pixel
410 * @param[in] X_STEP_ROW The incremental step row for the x coordinate (in pixels)
411 * @param[in] Y_STEP_ROW The incremental step row for the y coordinate (in pixels)
412 * @{
413 */
414#define LOAD_TEXTURE2D_ROW_1(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
415 BASENAME##0 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 0 * X_STEP_ROW), (Y_COORD + 0 * Y_STEP_ROW))
416
417#define LOAD_TEXTURE2D_ROW_2(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
418 LOAD_TEXTURE2D_ROW_1(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
419 BASENAME##1 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 1 * X_STEP_ROW), (Y_COORD + 1 * Y_STEP_ROW))
420
421#define LOAD_TEXTURE2D_ROW_3(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
422 LOAD_TEXTURE2D_ROW_2(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
423 BASENAME##2 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 2 * X_STEP_ROW), (Y_COORD + 2 * Y_STEP_ROW))
424
425#define LOAD_TEXTURE2D_ROW_4(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
426 LOAD_TEXTURE2D_ROW_3(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
427 BASENAME##3 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 3 * X_STEP_ROW), (Y_COORD + 3 * Y_STEP_ROW))
428
429#define LOAD_TEXTURE2D_ROW_5(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
430 LOAD_TEXTURE2D_ROW_4(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
431 BASENAME##4 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 4 * X_STEP_ROW), (Y_COORD + 4 * Y_STEP_ROW))
432
433#define LOAD_TEXTURE2D_ROW_6(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
434 LOAD_TEXTURE2D_ROW_5(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
435 BASENAME##5 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 5 * X_STEP_ROW), (Y_COORD + 5 * Y_STEP_ROW))
436
437#define LOAD_TEXTURE2D_ROW_7(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
438 LOAD_TEXTURE2D_ROW_6(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
439 BASENAME##6 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 6 * X_STEP_ROW), (Y_COORD + 6 * Y_STEP_ROW))
440
441#define LOAD_TEXTURE2D_ROW_8(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
442 LOAD_TEXTURE2D_ROW_7(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
443 BASENAME##7 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 7 * X_STEP_ROW), (Y_COORD + 7 * Y_STEP_ROW))
444
445#define LOAD_TEXTURE2D_ROW_9(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
446 LOAD_TEXTURE2D_ROW_8(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
447 BASENAME##8 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 8 * X_STEP_ROW), (Y_COORD + 8 * Y_STEP_ROW))
448
449#define LOAD_TEXTURE2D_ROW_10(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
450 LOAD_TEXTURE2D_ROW_9(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
451 BASENAME##9 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 9 * X_STEP_ROW), (Y_COORD + 9 * Y_STEP_ROW))
452
453#define LOAD_TEXTURE2D_ROW_11(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
454 LOAD_TEXTURE2D_ROW_10(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
455 BASENAME##A = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 10 * X_STEP_ROW), (Y_COORD + 10 * Y_STEP_ROW))
456
457#define LOAD_TEXTURE2D_ROW_12(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
458 LOAD_TEXTURE2D_ROW_11(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
459 BASENAME##B = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 11 * X_STEP_ROW), (Y_COORD + 11 * Y_STEP_ROW))
460
461#define LOAD_TEXTURE2D_ROW_13(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
462 LOAD_TEXTURE2D_ROW_12(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
463 BASENAME##C = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 12 * X_STEP_ROW), (Y_COORD + 12 * Y_STEP_ROW))
464
465#define LOAD_TEXTURE2D_ROW_14(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
466 LOAD_TEXTURE2D_ROW_13(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
467 BASENAME##D = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 13 * X_STEP_ROW), (Y_COORD + 13 * Y_STEP_ROW))
468
469#define LOAD_TEXTURE2D_ROW_15(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
470 LOAD_TEXTURE2D_ROW_14(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
471 BASENAME##E = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 14 * X_STEP_ROW), (Y_COORD + 14 * Y_STEP_ROW))
472
473#define LOAD_TEXTURE2D_ROW_16(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
474 LOAD_TEXTURE2D_ROW_15(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
475 BASENAME##F = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 15 * X_STEP_ROW), (Y_COORD + 15 * Y_STEP_ROW))
476/** @} */ // end of group LOAD_TEXTURE2D_ROW_n
477
478/** Load a 2D texture in unit of pixel. A pixel is made of 4 floating point values
479 * @name LOAD_TEXTURE2D
480 *
481 * Supported cases are M0=1,2,3,...,16 and N0=1
482 * The data to load is expected to have consecutive names for each row.
483 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
484 *
485 * @param[in] M0 The number of consecutive rows
486 * @param[in] N0 The number of consecutive pixels. Only 1, 2 and 4 are supported
487 * @param[in] DATA_TYPE The data type of the target
488 * @param[in] BASENAME The basename of the result variables
489 * @param[in] IMG The 2D OpenCL image object
490 * @param[in] X_COORD The x coordinate for the top-left pixel
491 * @param[in] Y_COORD The y coordinate for the top-left pixel
492 * @param[in] X_STEP_ROW The incremental step row for the x coordinate (in pixels)
493 * @param[in] Y_STEP_ROW The incremental step row for the y coordinate (in pixels)
494 * @{
495 */
496#define LOAD_TEXTURE2D_STR(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) LOAD_TEXTURE2D_ROW_##M0(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW)
497#define LOAD_TEXTURE2D(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) LOAD_TEXTURE2D_STR(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW)
498/** @} */ // end of group LOAD_TEXTURE2D
499
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]500/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1) passing the Y index for each row to be loaded.
501 * @name LOAD_ROW_INDIRECT_n
502 *
503 * @param[in] N0 The number of columns to load
504 * @param[in] DATA_TYPE The data type of variables
505 * @param[in] BASENAME The basename of the destination variables for the loaded rows
506 * @param[in] PTR The base pointer
507 * @param[in] OFFSET The offset within a row
508 * @param[in] STRIDE_Y The stride value in y-axis direction
509 * @param[in] Y The y-axis offset vector
510 * @param[in] Y_MASK The y-axis mask vector. If 0, forces BASENAMEn to 0
511 * @{
512 */
513#define LOAD_ROW_INDIRECT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
514 VEC_DATA_TYPE(DATA_TYPE, N0) \
515 BASENAME##0; \
516 if(Y_MASK##0 != 0) \
517 BASENAME##0 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##0 * STRIDE_Y)); \
518 else \
519 BASENAME##0 = 0;
520
521#define LOAD_ROW_INDIRECT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
522 LOAD_ROW_INDIRECT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
523 VEC_DATA_TYPE(DATA_TYPE, N0) \
524 BASENAME##1; \
525 if(Y_MASK##1 != 0) \
526 BASENAME##1 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##1 * STRIDE_Y)); \
527 else \
528 BASENAME##1 = 0;
529
530#define LOAD_ROW_INDIRECT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
531 LOAD_ROW_INDIRECT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
532 VEC_DATA_TYPE(DATA_TYPE, N0) \
533 BASENAME##2; \
534 if(Y_MASK##2 != 0) \
535 BASENAME##2 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##2 * STRIDE_Y)); \
536 else \
537 BASENAME##2 = 0;
538
539#define LOAD_ROW_INDIRECT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
540 LOAD_ROW_INDIRECT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
541 VEC_DATA_TYPE(DATA_TYPE, N0) \
542 BASENAME##3; \
543 if(Y_MASK##3 != 0) \
544 BASENAME##3 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##3 * STRIDE_Y)); \
545 else \
546 BASENAME##3 = 0;
547
548#define LOAD_ROW_INDIRECT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
549 LOAD_ROW_INDIRECT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
550 VEC_DATA_TYPE(DATA_TYPE, N0) \
551 BASENAME##4; \
552 if(Y_MASK##4 != 0) \
553 BASENAME##4 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##4 * STRIDE_Y)); \
554 else \
555 BASENAME##4 = 0;
556
557#define LOAD_ROW_INDIRECT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
558 LOAD_ROW_INDIRECT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
559 VEC_DATA_TYPE(DATA_TYPE, N0) \
560 BASENAME##5; \
561 if(Y_MASK##5 != 0) \
562 BASENAME##5 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##5 * STRIDE_Y)); \
563 else \
564 BASENAME##5 = 0;
565
566#define LOAD_ROW_INDIRECT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
567 LOAD_ROW_INDIRECT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
568 VEC_DATA_TYPE(DATA_TYPE, N0) \
569 BASENAME##6; \
570 if(Y_MASK##6 != 0) \
571 BASENAME##6 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##6 * STRIDE_Y)); \
572 else \
573 BASENAME##6 = 0;
574
575#define LOAD_ROW_INDIRECT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
576 LOAD_ROW_INDIRECT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
577 VEC_DATA_TYPE(DATA_TYPE, N0) \
578 BASENAME##7; \
579 if(Y_MASK##7 != 0) \
580 BASENAME##7 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##7 * STRIDE_Y)); \
581 else \
582 BASENAME##7 = 0;
583
584#define LOAD_ROW_INDIRECT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
585 LOAD_ROW_INDIRECT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
586 VEC_DATA_TYPE(DATA_TYPE, N0) \
587 BASENAME##8; \
588 if(Y_MASK##8 != 0) \
589 BASENAME##8 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##8 * STRIDE_Y)); \
590 else \
591 BASENAME##8 = 0;
592
593#define LOAD_ROW_INDIRECT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
594 LOAD_ROW_INDIRECT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
595 VEC_DATA_TYPE(DATA_TYPE, N0) \
596 BASENAME##9; \
597 if(Y_MASK##9 != 0) \
598 BASENAME##9 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##9 * STRIDE_Y)); \
599 else \
600 BASENAME##9 = 0;
601
602#define LOAD_ROW_INDIRECT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
603 LOAD_ROW_INDIRECT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
604 VEC_DATA_TYPE(DATA_TYPE, N0) \
605 BASENAME##A; \
606 if(Y_MASK##A != 0) \
607 BASENAME##A = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##A * STRIDE_Y)); \
608 else \
609 BASENAME##A = 0;
610
611#define LOAD_ROW_INDIRECT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
612 LOAD_ROW_INDIRECT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
613 VEC_DATA_TYPE(DATA_TYPE, N0) \
614 BASENAME##B; \
615 if(Y_MASK##B != 0) \
616 BASENAME##B = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##B * STRIDE_Y)); \
617 else \
618 BASENAME##B = 0;
619
620#define LOAD_ROW_INDIRECT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
621 LOAD_ROW_INDIRECT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
622 VEC_DATA_TYPE(DATA_TYPE, N0) \
623 BASENAME##C; \
624 if(Y_MASK##C != 0) \
625 BASENAME##C = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##C * STRIDE_Y)); \
626 else \
627 BASENAME##C = 0;
628
629#define LOAD_ROW_INDIRECT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
630 LOAD_ROW_INDIRECT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
631 VEC_DATA_TYPE(DATA_TYPE, N0) \
632 BASENAME##D; \
633 if(Y_MASK##D != 0) \
634 BASENAME##D = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##D * STRIDE_Y)); \
635 else \
636 BASENAME##D = 0;
637
638#define LOAD_ROW_INDIRECT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
639 LOAD_ROW_INDIRECT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
640 VEC_DATA_TYPE(DATA_TYPE, N0) \
641 BASENAME##E; \
642 if(Y_MASK##E != 0) \
643 BASENAME##E = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##E * STRIDE_Y)); \
644 else \
645 BASENAME##E = 0;
646
647#define LOAD_ROW_INDIRECT_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
648 LOAD_ROW_INDIRECT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) \
649 VEC_DATA_TYPE(DATA_TYPE, N0) \
650 BASENAME##F; \
651 if(Y_MASK##F != 0) \
652 BASENAME##F = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + Y##F * STRIDE_Y)); \
653 else \
654 BASENAME##F = 0;
655
656/** Load blocks (consecutive rows and columns) with Y offset.
657 * @name LOAD_BLOCK_INDIRECT
658 *
659 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
660 * The data to load is expected to have consecutive names for each row.
661 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
662 * The Z offset is expected to have consecutive names.
663 * E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
664 *
665 * @param[in] M0 The number of consecutive rows
666 * @param[in] N0 The number of consecutive columns
667 * @param[in] DATA_TYPE The data type of the target
668 * @param[in] BASENAME The basename of the result variables
669 * @param[in] PTR The base pointer for the data
670 * @param[in] OFFSET The offset within a row
671 * @param[in] STRIDE_Y The stride in y-axis direction
672 * @param[in] Y The y-axis offset vector
673 * @param[in] Y_MASK The y-axis mask vector. If 0, forces BASENAMEn to 0
674 * @{
675 */
676#define LOAD_BLOCK_INDIRECT_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) LOAD_ROW_INDIRECT_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK)
677#define LOAD_BLOCK_INDIRECT(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK) LOAD_BLOCK_INDIRECT_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Y, Y_MASK)
678
Michele Di Giorgiob54ba282020-01-14 15:31:55 +0000[diff] [blame]679/** Loads the elements from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
680 * @name LOAD_ELEMENT_n
681 *
682 * @param[in] N0 The number of rows to load
683 * @param[in] DATA_TYPE The data type of variables
684 * @param[in] BASENAME The basename of the destination variables for the loaded rows
685 * @param[in] PTR The base pointer
686 * @param[in] OFFSET The offset within a row
687 * @param[in] STRIDE_Y The stride value in y-axis direction
688 * @{
689 */
690#define LOAD_ELEMENT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
691 VEC_DATA_TYPE(DATA_TYPE, N0) \
692 BASENAME##0 = *((__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y));
693
694#define LOAD_ELEMENT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
695 LOAD_ELEMENT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
696 VEC_DATA_TYPE(DATA_TYPE, N0) \
697 BASENAME##1 = *((__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y));
698
699#define LOAD_ELEMENT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
700 LOAD_ELEMENT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
701 VEC_DATA_TYPE(DATA_TYPE, N0) \
702 BASENAME##2 = *((__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y));
703
704#define LOAD_ELEMENT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
705 LOAD_ELEMENT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
706 VEC_DATA_TYPE(DATA_TYPE, N0) \
707 BASENAME##3 = *((__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y));
708
709#define LOAD_ELEMENT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
710 LOAD_ELEMENT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
711 VEC_DATA_TYPE(DATA_TYPE, N0) \
712 BASENAME##4 = *((__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y));
713
714#define LOAD_ELEMENT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
715 LOAD_ELEMENT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
716 VEC_DATA_TYPE(DATA_TYPE, N0) \
717 BASENAME##5 = *((__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y));
718
719#define LOAD_ELEMENT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
720 LOAD_ELEMENT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
721 VEC_DATA_TYPE(DATA_TYPE, N0) \
722 BASENAME##6 = *((__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y));
723
724#define LOAD_ELEMENT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
725 LOAD_ELEMENT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
726 VEC_DATA_TYPE(DATA_TYPE, N0) \
727 BASENAME##7 = *((__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y));
728
729#define LOAD_ELEMENT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
730 LOAD_ELEMENT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
731 VEC_DATA_TYPE(DATA_TYPE, N0) \
732 BASENAME##8 = *((__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y));
733
734#define LOAD_ELEMENT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
735 LOAD_ELEMENT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
736 VEC_DATA_TYPE(DATA_TYPE, N0) \
737 BASENAME##9 = *((__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y));
738
739#define LOAD_ELEMENT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
740 LOAD_ELEMENT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
741 VEC_DATA_TYPE(DATA_TYPE, N0) \
742 BASENAME##A = *((__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y));
743
744#define LOAD_ELEMENT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
745 LOAD_ELEMENT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
746 VEC_DATA_TYPE(DATA_TYPE, N0) \
747 BASENAME##B = *((__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y));
748
749#define LOAD_ELEMENT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
750 LOAD_ELEMENT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
751 VEC_DATA_TYPE(DATA_TYPE, N0) \
752 BASENAME##C = *((__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y));
753
754#define LOAD_ELEMENT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
755 LOAD_ELEMENT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
756 VEC_DATA_TYPE(DATA_TYPE, N0) \
757 BASENAME##D = *((__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y));
758
759#define LOAD_ELEMENT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
760 LOAD_ELEMENT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
761 VEC_DATA_TYPE(DATA_TYPE, N0) \
762 BASENAME##E = *((__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y));
763
764#define LOAD_ELEMENT_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
765 LOAD_ELEMENT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
766 VEC_DATA_TYPE(DATA_TYPE, N0) \
767 BASENAME##F = *((__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y));
768
769/** @}*/ // end of group LOAD_ELEMENT_n
770
771/** Load Scalar as Vector (consecutive elements).
772 * @name LOAD_SCALAR_AS_VECTOR
773 *
774 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
775 * The data to load is expected to have consecutive names for each row.
776 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
777 *
778 * @param[in] M0 The number of consecutive rows
779 * @param[in] N0 The number of consecutive columns
780 * @param[in] DATA_TYPE The data type of the target
781 * @param[in] BASENAME The basename of the result variables
782 * @param[in] PTR The base pointer for the data
783 * @param[in] OFFSET The offset within a row
784 * @param[in] STRIDE_Y The stride in y-axis direction
785 * @{
786 */
787#define LOAD_SCALAR_AS_VECTOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) LOAD_ELEMENT_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y)
788#define LOAD_SCALAR_AS_VECTOR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) LOAD_SCALAR_AS_VECTOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y)
789/** @} */ // end of group LOAD_SCALAR_AS_VECTOR
790
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]791/** Basic macros to calculate Z offset values from Z0 to Zn-1
792 * @name CALCULATE_Z_OFFSET_n
793 *
794 * @param[in] M0 The number of offset values to calculate
795 * @param[in] DATA_TYPE The data type of the results
796 * @param[in] Z The basename of the result variables
797 * @param[in] Y The work-itme ID of y-axis
798 * @param[in] HEIGHT_GEMM3D The height of GEMM3D
799 * @param[in] DEPTH_GEMM3D The depth of GEMM3D
800 * @param[in] CROSS_PLANE_PAD The padding required for plane changes accross the z-dimension
801 * @param[in] STRIDE_Y The stride value in y-axis direction
802 *
803 * @{
804 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]805#define CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]806 Z##0 = (0 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]807 Z##0 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##0); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]808 Z##0 *= (CROSS_PLANE_PAD * STRIDE_Y);
809
810#define CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
811 CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]812 Z##1 = (1 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]813 Z##1 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##1); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]814 Z##1 *= (CROSS_PLANE_PAD * STRIDE_Y);
815
816#define CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
817 CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]818 Z##2 = (2 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]819 Z##2 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##2); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]820 Z##2 *= (CROSS_PLANE_PAD * STRIDE_Y);
821
822#define CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
823 CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]824 Z##3 = (3 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]825 Z##3 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##3); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]826 Z##3 *= (CROSS_PLANE_PAD * STRIDE_Y);
827
828#define CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
829 CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]830 Z##4 = (4 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]831 Z##4 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##4); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]832 Z##4 *= (CROSS_PLANE_PAD * STRIDE_Y);
833
834#define CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
835 CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]836 Z##5 = (5 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]837 Z##5 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##5); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]838 Z##5 *= (CROSS_PLANE_PAD * STRIDE_Y);
839
840#define CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
841 CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]842 Z##6 = (6 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]843 Z##6 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##6); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]844 Z##6 *= (CROSS_PLANE_PAD * STRIDE_Y);
845
846#define CALCULATE_Z_OFFSET_8(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
847 CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodiceff1fe3e2021-01-02 09:58:51 +0000[diff] [blame]848 Z##7 = (7 + (DATA_TYPE)(Y)) / (DATA_TYPE)HEIGHT_GEMM3D; \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]849 Z##7 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##7); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]850 Z##7 *= (CROSS_PLANE_PAD * STRIDE_Y);
851
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]852/** @} */ // end of group CALCULATE_Z_OFFSET_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]853
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]854/** Calculate Z offset values from Z0 to Zn-1
855 * @name CALCULATE_Z_OFFSET
856 *
857 * The Z offsets are expected to have consecutive names.
858 * E.g., for M0=3 and Z=zin, the expected names of Z offsets are zin1, zin2, zin3.
859 * Note that, CROSS_PLANE_PAD (cross plain padding) is required to take into account
860 * the possible cross plane paddings in case of the plance changes across the z-dimension.
861 *
862 * <!--
863 * | |
864 * | plane0 |
865 * | |
866 * |__________________|
867 * |******************|
868 * | cross_plane_pad |
869 * |******************|
870 * | |
871 * | plane1 |
872 * | |
873 * |__________________|
874 * -->
875 *
876 * @param[in] M0 The number of offset values to calculate
877 * @param[in] DATA_TYPE The data type of the results
878 * @param[in] Z The basename of the result variables
879 * @param[in] Y The work-itme ID of y-axis
880 * @param[in] HEIGHT_GEMM3D The height of GEMM3D
881 * @param[in] DEPTH_GEMM3D The depth of GEMM3D
882 * @param[in] CROSS_PLANE_PAD The padding required for plane changes accross the z-dimension
883 * @param[in] STRIDE_Y The stride value in y-axis direction
884 * @{
885 */
886#define CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_##M0(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
887#define CALCULATE_Z_OFFSET(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
888/** @} */ // end of group CALCULATE_Z_OFFSET
889
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]890/** Scale the rows in the given variables (BASENAME0 to BASENAMEn-1)
891 * @name SCALE_ROW_n
892 *
893 * @param[in] DATA_TYPE The data type of the variables
894 * @param[in] BASENAME The basename of the variables
895 * @param[in] SCALE The scale factor
896 * @{
897 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]898#define SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]899 BASENAME##0 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]900
901#define SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
902 SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]903 BASENAME##1 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]904
905#define SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
906 SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]907 BASENAME##2 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]908
909#define SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
910 SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]911 BASENAME##3 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]912
913#define SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
914 SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]915 BASENAME##4 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]916
917#define SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
918 SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]919 BASENAME##5 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]920
921#define SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
922 SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]923 BASENAME##6 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]924
925#define SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
926 SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]927 BASENAME##7 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]928
929#define SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
930 SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]931 BASENAME##8 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]932
933#define SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
934 SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]935 BASENAME##9 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]936
937#define SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
938 SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]939 BASENAME##A *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]940
941#define SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
942 SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]943 BASENAME##B *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]944
945#define SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
946 SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]947 BASENAME##C *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]948
949#define SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
950 SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]951 BASENAME##D *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]952
953#define SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
954 SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]955 BASENAME##E *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]956
957#define SCALE_ROW_16(DATA_TYPE, BASENAME, SCALE) \
958 SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]959 BASENAME##F *= (DATA_TYPE)SCALE;
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]960/** @} */ // end of group SCALE_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]961
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]962/** Scale elements stored in a block (BASENAME)
963 * @name SCALE_BLOCK
964 *
965 * Supported cases are N=1,2,3,...,16
966 *
967 * @param[in] N The number of rows in the block
968 * @param[in] DATA_TYPE The data type of the block
969 * @param[in] BASENAME The basename of the block
970 * @param[in] SCALE The scale factor
971 * @{
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]972 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]973#define SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE) SCALE_ROW_##N(DATA_TYPE, BASENAME, SCALE)
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]974#define SCALE_BLOCK(N, DATA_TYPE, BASENAME, SCALE) SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]975/** @} */ // end of group SCALE_BLOCK
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]976
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]977/** Create a new vector containing the values at the given index for a set of given vectors
978 * @name COLUMN_VECTORn
979 *
980 * @param[in] IDX_COL The index value
981 * @param[in] BASENAME The basename of the destination vectors
982 * @param[in] X The basename of the source vectors
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]983 * @param[in] TYPE The data type of the destination vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]984 * @{
985 */
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]986#define COLUMN_VECTOR1(IDX_COL, BASENAME, X, TYPE) \
987 TYPE BASENAME##IDX_COL = (TYPE)((X##0).s##IDX_COL);
988#define COLUMN_VECTOR2(IDX_COL, BASENAME, X, TYPE) \
989 VEC_DATA_TYPE(TYPE, 2) \
990 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 2))((X##0).s##IDX_COL, (X##1).s##IDX_COL);
991#define COLUMN_VECTOR3(IDX_COL, BASENAME, X, TYPE) \
992 VEC_DATA_TYPE(TYPE, 3) \
993 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 3))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL);
994#define COLUMN_VECTOR4(IDX_COL, BASENAME, X, TYPE) \
995 VEC_DATA_TYPE(TYPE, 4) \
996 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 4))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL);
997#define COLUMN_VECTOR8(IDX_COL, BASENAME, X, TYPE) \
998 VEC_DATA_TYPE(TYPE, 8) \
999 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 8))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL, (X##4).s##IDX_COL, (X##5).s##IDX_COL, (X##6).s##IDX_COL, (X##7).s##IDX_COL);
1000#define COLUMN_VECTOR16(IDX_COL, BASENAME, X, TYPE) \
1001 VEC_DATA_TYPE(TYPE, 16) \
1002 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 16))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL, (X##4).s##IDX_COL, (X##5).s##IDX_COL, (X##6).s##IDX_COL, (X##7).s##IDX_COL, (X##8).s##IDX_COL, (X##9).s##IDX_COL, (X##A).s##IDX_COL, (X##B).s##IDX_COL, (X##C).s##IDX_COL, (X##D).s##IDX_COL, (X##E).s##IDX_COL, (X##F).s##IDX_COL);
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1003/** @} */ // end of group COLUMN_VECTORn
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1004
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1005/** Create a new vector containing the values at the given index. Utility macros for transposing a colum-vector
1006 * @name COLUMN_VECTOR_SCALARn
1007 *
1008 * @param[in] IDX_COL The index value
1009 * @param[in] BASENAME The basename of the destination vectors
1010 * @param[in] X The basename of the source vectors
1011 * @param[in] TYPE The data type of the destination vectors
1012 * @{
1013 */
1014#define COLUMN_VECTOR_SCALAR1(IDX_COL, BASENAME, X, TYPE) \
1015 TYPE BASENAME##IDX_COL = (TYPE)((X##0));
1016#define COLUMN_VECTOR_SCALAR2(IDX_COL, BASENAME, X, TYPE) \
1017 VEC_DATA_TYPE(TYPE, 2) \
1018 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 2))((X##0), (X##1));
1019#define COLUMN_VECTOR_SCALAR3(IDX_COL, BASENAME, X, TYPE) \
1020 VEC_DATA_TYPE(TYPE, 3) \
1021 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 3))((X##0), (X##1), (X##2));
1022#define COLUMN_VECTOR_SCALAR4(IDX_COL, BASENAME, X, TYPE) \
1023 VEC_DATA_TYPE(TYPE, 4) \
1024 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 4))((X##0), (X##1), (X##2), (X##3));
1025#define COLUMN_VECTOR_SCALAR8(IDX_COL, BASENAME, X, TYPE) \
1026 VEC_DATA_TYPE(TYPE, 8) \
1027 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 8))((X##0), (X##1), (X##2), (X##3), (X##4), (X##5), (X##6), (X##7));
1028#define COLUMN_VECTOR_SCALAR16(IDX_COL, BASENAME, X, TYPE) \
1029 VEC_DATA_TYPE(TYPE, 16) \
1030 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 16))((X##0), (X##1), (X##2), (X##3), (X##4), (X##5), (X##6), (X##7), (X##8), (X##9), (X##A), (X##B), (X##C), (X##D), (X##E), (X##F));
1031/** @} */ // end of group COLUMN_VECTORn
1032
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1033/** Create transposed vectors of the given vectors
1034 * @name TRANSPOSE_K0Xn
1035 *
1036 * @param[in] K0 The size of the source vectors
1037 * @param[in] BASENAME The basename of transposed vectors
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1038 * @param[in] BS The basename of source vectors for transposition
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1039 * @param[in] TYPE The data type of the transposed vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1040 * @{
1041 */
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1042#define TRANSPOSE_K0X1(K0, BASENAME, BS, TYPE) \
1043 COLUMN_VECTOR_SCALAR(K0, 0, BASENAME, BS, TYPE);
1044#define TRANSPOSE_K0X2(K0, BASENAME, BS, TYPE) \
1045 COLUMN_VECTOR(K0, 0, BASENAME, BS, TYPE); \
1046 COLUMN_VECTOR(K0, 1, BASENAME, BS, TYPE);
1047#define TRANSPOSE_K0X3(K0, BASENAME, BS, TYPE) \
1048 TRANSPOSE_K0X2(K0, BASENAME, BS, TYPE); \
1049 COLUMN_VECTOR(K0, 2, BASENAME, BS, TYPE);
1050#define TRANSPOSE_K0X4(K0, BASENAME, BS, TYPE) \
1051 TRANSPOSE_K0X3(K0, BASENAME, BS, TYPE); \
1052 COLUMN_VECTOR(K0, 3, BASENAME, BS, TYPE);
1053#define TRANSPOSE_K0X8(K0, BASENAME, BS, TYPE) \
1054 TRANSPOSE_K0X4(K0, BASENAME, BS, TYPE); \
1055 COLUMN_VECTOR(K0, 4, BASENAME, BS, TYPE); \
1056 COLUMN_VECTOR(K0, 5, BASENAME, BS, TYPE); \
1057 COLUMN_VECTOR(K0, 6, BASENAME, BS, TYPE); \
1058 COLUMN_VECTOR(K0, 7, BASENAME, BS, TYPE);
1059#define TRANSPOSE_K0X16(K0, BASENAME, BS, TYPE) \
1060 TRANSPOSE_K0X8(K0, BASENAME, BS, TYPE); \
1061 COLUMN_VECTOR(K0, 8, BASENAME, BS, TYPE); \
1062 COLUMN_VECTOR(K0, 9, BASENAME, BS, TYPE); \
1063 COLUMN_VECTOR(K0, A, BASENAME, BS, TYPE); \
1064 COLUMN_VECTOR(K0, B, BASENAME, BS, TYPE); \
1065 COLUMN_VECTOR(K0, C, BASENAME, BS, TYPE); \
1066 COLUMN_VECTOR(K0, D, BASENAME, BS, TYPE); \
1067 COLUMN_VECTOR(K0, E, BASENAME, BS, TYPE); \
1068 COLUMN_VECTOR(K0, F, BASENAME, BS, TYPE);
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1069
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1070/** @} */ // end of group TRANSPOSE_K0Xn
1071
1072/** Create column vectors to contain the values at the given index for a set of given vectors
1073 *
1074 * @param[in] K0 The number of source vectors
1075 * @param[in] IDX_COL The index value
1076 * @param[in] BASENAME The basename of the destination vectors
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1077 * @param[in] BS The basename of the source vectors
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1078 * @param[in] TYPE The data type of the destination vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1079 */
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1080#define COLUMN_VECTOR(K0, IDX_COL, BASENAME, BS, TYPE) \
1081 CONCAT(COLUMN_VECTOR, K0) \
1082 (IDX_COL, BASENAME, BS, TYPE);
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1083
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1084/** Create column vectors to contain the values at the given index. Utility macro for transposing a column-vector
1085 *
1086 * @param[in] K0 The number of source vectors
1087 * @param[in] IDX_COL The index value
1088 * @param[in] BASENAME The basename of the destination vectors
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1089 * @param[in] BS The basename of the source vectors
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1090 * @param[in] TYPE The data type of the destination vectors
1091 */
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1092#define COLUMN_VECTOR_SCALAR(K0, IDX_COL, BASENAME, BS, TYPE) \
1093 CONCAT(COLUMN_VECTOR_SCALAR, K0) \
1094 (IDX_COL, BASENAME, BS, TYPE);
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1095
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1096/** Create transposed vectors form the given source vectors
1097 *
1098 * @param[in] K0 The size of source vectors
1099 * @param[in] N0 The number of source vectors
1100 * @param[in] BASENAME The basename of transposed vectors
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1101 * @param[in] BS The basename of source vectors for transposition
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1102 * @param[in] TYPE The data type of the transposed vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1103 *
1104 */
SiCongLib5323cf2021-03-04 15:53:31 +0000[diff] [blame]1105#define TRANSPOSE_K0XN0(K0, N0, BASENAME, BS, TYPE) \
1106 CONCAT(TRANSPOSE_K0X, N0) \
1107 (K0, BASENAME, BS, TYPE);
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1108
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1109/** Add the variables (BIAS0 to BIASn-1) to the others (BASENAME0 to BASENAMEn-1)
1110 * @name ADD_ROW_n
1111 *
1112 * @param[in] BASENAME The basename of the destination variables
1113 * @param[in] BIAS The basename of the added variables
1114 * @{
1115 */
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1116#define ADD_ROW_1(BASENAME, BIAS) \
1117 BASENAME##0 += BIAS##0;
1118
1119#define ADD_ROW_2(BASENAME, BIAS) \
1120 ADD_ROW_1(BASENAME, BIAS) \
1121 BASENAME##1 += BIAS##1;
1122
1123#define ADD_ROW_3(BASENAME, BIAS) \
1124 ADD_ROW_2(BASENAME, BIAS) \
1125 BASENAME##2 += BIAS##2;
1126
1127#define ADD_ROW_4(BASENAME, BIAS) \
1128 ADD_ROW_3(BASENAME, BIAS) \
1129 BASENAME##3 += BIAS##3;
1130
1131#define ADD_ROW_5(BASENAME, BIAS) \
1132 ADD_ROW_4(BASENAME, BIAS) \
1133 BASENAME##4 += BIAS##4;
1134
1135#define ADD_ROW_6(BASENAME, BIAS) \
1136 ADD_ROW_5(BASENAME, BIAS) \
1137 BASENAME##5 += BIAS##5;
1138
1139#define ADD_ROW_7(BASENAME, BIAS) \
1140 ADD_ROW_6(BASENAME, BIAS) \
1141 BASENAME##6 += BIAS##6;
1142
1143#define ADD_ROW_8(BASENAME, BIAS) \
1144 ADD_ROW_7(BASENAME, BIAS) \
1145 BASENAME##7 += BIAS##7;
1146
1147#define ADD_ROW_9(BASENAME, BIAS) \
1148 ADD_ROW_8(BASENAME, BIAS) \
1149 BASENAME##8 += BIAS##8;
1150
1151#define ADD_ROW_10(BASENAME, BIAS) \
1152 ADD_ROW_9(BASENAME, BIAS) \
1153 BASENAME##9 += BIAS##9;
1154
1155#define ADD_ROW_11(BASENAME, BIAS) \
1156 ADD_ROW_10(BASENAME, BIAS) \
1157 BASENAME##A += BIAS##A;
1158
1159#define ADD_ROW_12(BASENAME, BIAS) \
1160 ADD_ROW_11(BASENAME, BIAS) \
1161 BASENAME##B += BIAS##B;
1162
1163#define ADD_ROW_13(BASENAME, BIAS) \
1164 ADD_ROW_12(BASENAME, BIAS) \
1165 BASENAME##C += BIAS##C;
1166
1167#define ADD_ROW_14(BASENAME, BIAS) \
1168 ADD_ROW_13(BASENAME, BIAS) \
1169 BASENAME##D += BIAS##D;
1170
1171#define ADD_ROW_15(BASENAME, BIAS) \
1172 ADD_ROW_14(BASENAME, BIAS) \
1173 BASENAME##E += BIAS##E;
1174
1175#define ADD_ROW_16(BASENAME, BIAS) \
1176 ADD_ROW_15(BASENAME, BIAS) \
1177 BASENAME##F += BIAS##F;
1178
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1179/** @} */ // end of group ADD_ROW_n
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1180
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1181/** Add the block (BIAS) to another block (BASENAME)
1182 * @name ADD_BLOCK
1183 *
1184 * Supported cases are N=1,2,3,...,16
1185 *
1186 * @param[in] N The number of vectors in the block
1187 * @param[in] BASENAME The basename of the destination variables
1188 * @param[in] BIAS The basename of the added variables
1189 * @{
1190 */
1191#define ADD_BLOCK_STR(N, BASENAME, BIAS) ADD_ROW_##N(BASENAME, BIAS)
1192#define ADD_BLOCK(N, BASENAME, BIAS) ADD_BLOCK_STR(N, BASENAME, BIAS)
1193/** @} */ // end of group ADD_BLOCK
1194
1195/** Broadcast (add single value) to the each element of the destination variables
1196 * @name ADD_ROW_BROADCAST_n
1197 *
1198 * @param[in] BASENAME The basename of the destination variables
1199 * @param[in] BIAS The variable containing the value to add
1200 * @{
1201 */
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1202#define ADD_ROW_BROADCAST_1(BASENAME, BIAS) \
1203 BASENAME##0 += BIAS;
1204
1205#define ADD_ROW_BROADCAST_2(BASENAME, BIAS) \
1206 ADD_ROW_BROADCAST_1(BASENAME, BIAS) \
1207 BASENAME##1 += BIAS;
1208
1209#define ADD_ROW_BROADCAST_3(BASENAME, BIAS) \
1210 ADD_ROW_BROADCAST_2(BASENAME, BIAS) \
1211 BASENAME##2 += BIAS;
1212
1213#define ADD_ROW_BROADCAST_4(BASENAME, BIAS) \
1214 ADD_ROW_BROADCAST_3(BASENAME, BIAS) \
1215 BASENAME##3 += BIAS;
1216
1217#define ADD_ROW_BROADCAST_5(BASENAME, BIAS) \
1218 ADD_ROW_BROADCAST_4(BASENAME, BIAS) \
1219 BASENAME##4 += BIAS;
1220
1221#define ADD_ROW_BROADCAST_6(BASENAME, BIAS) \
1222 ADD_ROW_BROADCAST_5(BASENAME, BIAS) \
1223 BASENAME##5 += BIAS;
1224
1225#define ADD_ROW_BROADCAST_7(BASENAME, BIAS) \
1226 ADD_ROW_BROADCAST_6(BASENAME, BIAS) \
1227 BASENAME##6 += BIAS;
1228
1229#define ADD_ROW_BROADCAST_8(BASENAME, BIAS) \
1230 ADD_ROW_BROADCAST_7(BASENAME, BIAS) \
1231 BASENAME##7 += BIAS;
1232
1233#define ADD_ROW_BROADCAST_9(BASENAME, BIAS) \
1234 ADD_ROW_BROADCAST_8(BASENAME, BIAS) \
1235 BASENAME##8 += BIAS;
1236
1237#define ADD_ROW_BROADCAST_10(BASENAME, BIAS) \
1238 ADD_ROW_BROADCAST_9(BASENAME, BIAS) \
1239 BASENAME##9 += BIAS;
1240
1241#define ADD_ROW_BROADCAST_11(BASENAME, BIAS) \
1242 ADD_ROW_BROADCAST_10(BASENAME, BIAS) \
1243 BASENAME##A += BIAS;
1244
1245#define ADD_ROW_BROADCAST_12(BASENAME, BIAS) \
1246 ADD_ROW_BROADCAST_11(BASENAME, BIAS) \
1247 BASENAME##B += BIAS;
1248
1249#define ADD_ROW_BROADCAST_13(BASENAME, BIAS) \
1250 ADD_ROW_BROADCAST_12(BASENAME, BIAS) \
1251 BASENAME##C += BIAS;
1252
1253#define ADD_ROW_BROADCAST_14(BASENAME, BIAS) \
1254 ADD_ROW_BROADCAST_13(BASENAME, BIAS) \
1255 BASENAME##D += BIAS;
1256
1257#define ADD_ROW_BROADCAST_15(BASENAME, BIAS) \
1258 ADD_ROW_BROADCAST_14(BASENAME, BIAS) \
1259 BASENAME##E += BIAS;
1260
1261#define ADD_ROW_BROADCAST_16(BASENAME, BIAS) \
1262 ADD_ROW_BROADCAST_15(BASENAME, BIAS) \
1263 BASENAME##F += BIAS;
1264
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1265/** Broadcast (add a value) to the each element of the destination block (BASENAME)
1266 * @name ADD_BLOCK_BROADCAST
1267 *
1268 * Supported cases are N=1,2,3,...,16.
1269 *
1270 * @param[in] N The number of vectors in the block
1271 * @param[in] BASENAME The basename of the destination variables
1272 * @param[in] BIAS The variable containing the value to add
1273 * @{
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1274 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1275#define ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS) ADD_ROW_BROADCAST_##N(BASENAME, BIAS)
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1276#define ADD_BLOCK_BROADCAST(N, BASENAME, BIAS) ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1277/** @} */ // end of group ADD_BLOCK_BROADCAST
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1278
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1279/** Apply activation to the given variables
1280 * @name ACTIVATION_ROW_n
1281 *
1282 * @param[in] ACTIVATION_TYPE The type of the activation
1283 * @param[in] DATA_TYPE The data type of the vectors
1284 * @param[in] BASENAME The basename of the variables
1285 * @param[in] A_VAL Additional value required by the activation
1286 * @param[in] B_VAL Additional value required by the activation
1287 * @{
1288 */
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1289#define ACTIVATION_ROW_1(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1290 BASENAME##0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##0, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1291
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1292#define ACTIVATION_ROW_2(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1293 ACTIVATION_ROW_1(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1294 BASENAME##1 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##1, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1295
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1296#define ACTIVATION_ROW_3(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1297 ACTIVATION_ROW_2(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1298 BASENAME##2 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##2, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1299
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1300#define ACTIVATION_ROW_4(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1301 ACTIVATION_ROW_3(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1302 BASENAME##3 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##3, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1303
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1304#define ACTIVATION_ROW_5(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1305 ACTIVATION_ROW_4(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1306 BASENAME##4 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##4, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1307
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1308#define ACTIVATION_ROW_6(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1309 ACTIVATION_ROW_5(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1310 BASENAME##5 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##5, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1311
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1312#define ACTIVATION_ROW_7(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1313 ACTIVATION_ROW_6(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1314 BASENAME##6 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##6, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1315
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1316#define ACTIVATION_ROW_8(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1317 ACTIVATION_ROW_7(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1318 BASENAME##7 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##7, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1319
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1320#define ACTIVATION_ROW_9(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1321 ACTIVATION_ROW_8(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1322 BASENAME##8 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##8, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1323
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1324#define ACTIVATION_ROW_10(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1325 ACTIVATION_ROW_9(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1326 BASENAME##9 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##9, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1327
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1328#define ACTIVATION_ROW_11(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1329 ACTIVATION_ROW_10(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1330 BASENAME##A = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##A, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1331
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1332#define ACTIVATION_ROW_12(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1333 ACTIVATION_ROW_11(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1334 BASENAME##B = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##B, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1335
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1336#define ACTIVATION_ROW_13(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1337 ACTIVATION_ROW_12(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1338 BASENAME##C = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##C, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1339
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1340#define ACTIVATION_ROW_14(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1341 ACTIVATION_ROW_13(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1342 BASENAME##D = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##D, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1343
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1344#define ACTIVATION_ROW_15(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1345 ACTIVATION_ROW_14(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1346 BASENAME##E = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##E, A_VAL, B_VAL);
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1347
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1348#define ACTIVATION_ROW_16(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1349 ACTIVATION_ROW_15(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) \
1350 BASENAME##F = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME##F, A_VAL, B_VAL);
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1351/** @} */ // end of group ACTIVATION_ROW_n
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1352
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1353/** Apply activation to a block (BASENAME)
1354 * @name ACTIVATION_BLOCK
1355 *
1356 * Supported cases are N=1,2,3,...,16.
1357 *
1358 * @param[in] N The number of vectors in the block
1359 * @param[in] ACTIVATION_TYPE The type of the activation
1360 * @param[in] DATA_TYPE The data type of the vectors
1361 * @param[in] BASENAME The basename of the variables
1362 * @param[in] A_VAL Additional value required by the activation
1363 * @param[in] B_VAL Additional value required by the activation
1364 * @{
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1365 */
Giorgio Arenad056e572020-10-12 11:53:51 +0100[diff] [blame]1366#define ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) ACTIVATION_ROW_##N(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL)
1367#define ACTIVATION_BLOCK(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL) ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, BASENAME, A_VAL, B_VAL)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1368/** @} */ // end of group ACTIVATION_BLOCK
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1369
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1370/** Apply convert_<data_type> to the given variables
1371 * @name CONVERT_ROW_n
1372 *
1373 * @param[in] N The size of the vectors
1374 * @param[in] DATA_TYPE The data type of the vectors
1375 * @param[in] BASENAME_SRC The basename of the source variables
1376 * @param[in] BASENAME_DST The basename of the destination variables
1377 */
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1378#define CONVERT_ROW_1(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1379 VEC_DATA_TYPE(DATA_TYPE, N) \
1380 BASENAME_DST##0 = CONVERT(BASENAME_SRC##0, VEC_DATA_TYPE(DATA_TYPE, N));
1381
1382#define CONVERT_ROW_2(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1383 CONVERT_ROW_1(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1384 VEC_DATA_TYPE(DATA_TYPE, N) \
1385 BASENAME_DST##1 = CONVERT(BASENAME_SRC##1, VEC_DATA_TYPE(DATA_TYPE, N));
1386
1387#define CONVERT_ROW_3(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1388 CONVERT_ROW_2(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1389 VEC_DATA_TYPE(DATA_TYPE, N) \
1390 BASENAME_DST##2 = CONVERT(BASENAME_SRC##2, VEC_DATA_TYPE(DATA_TYPE, N));
1391
1392#define CONVERT_ROW_4(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1393 CONVERT_ROW_3(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1394 VEC_DATA_TYPE(DATA_TYPE, N) \
1395 BASENAME_DST##3 = CONVERT(BASENAME_SRC##3, VEC_DATA_TYPE(DATA_TYPE, N));
1396
1397#define CONVERT_ROW_5(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1398 CONVERT_ROW_4(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1399 VEC_DATA_TYPE(DATA_TYPE, N) \
1400 BASENAME_DST##4 = CONVERT(BASENAME_SRC##4, VEC_DATA_TYPE(DATA_TYPE, N));
1401
1402#define CONVERT_ROW_6(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1403 CONVERT_ROW_5(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1404 VEC_DATA_TYPE(DATA_TYPE, N) \
1405 BASENAME_DST##5 = CONVERT(BASENAME_SRC##5, VEC_DATA_TYPE(DATA_TYPE, N));
1406
1407#define CONVERT_ROW_7(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1408 CONVERT_ROW_6(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1409 VEC_DATA_TYPE(DATA_TYPE, N) \
1410 BASENAME_DST##6 = CONVERT(BASENAME_SRC##6, VEC_DATA_TYPE(DATA_TYPE, N));
1411
1412#define CONVERT_ROW_8(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1413 CONVERT_ROW_7(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1414 VEC_DATA_TYPE(DATA_TYPE, N) \
1415 BASENAME_DST##7 = CONVERT(BASENAME_SRC##7, VEC_DATA_TYPE(DATA_TYPE, N));
1416
1417#define CONVERT_ROW_9(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1418 CONVERT_ROW_8(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1419 VEC_DATA_TYPE(DATA_TYPE, N) \
1420 BASENAME_DST##8 = CONVERT(BASENAME_SRC##8, VEC_DATA_TYPE(DATA_TYPE, N));
1421
1422#define CONVERT_ROW_10(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1423 CONVERT_ROW_9(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1424 VEC_DATA_TYPE(DATA_TYPE, N) \
1425 BASENAME_DST##9 = CONVERT(BASENAME_SRC##9, VEC_DATA_TYPE(DATA_TYPE, N));
1426
1427#define CONVERT_ROW_11(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1428 CONVERT_ROW_10(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1429 VEC_DATA_TYPE(DATA_TYPE, N) \
1430 BASENAME_DST##A = CONVERT(BASENAME_SRC##A, VEC_DATA_TYPE(DATA_TYPE, N));
1431
1432#define CONVERT_ROW_12(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1433 CONVERT_ROW_11(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1434 VEC_DATA_TYPE(DATA_TYPE, N) \
1435 BASENAME_DST##B = CONVERT(BASENAME_SRC##B, VEC_DATA_TYPE(DATA_TYPE, N));
1436
1437#define CONVERT_ROW_13(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1438 CONVERT_ROW_12(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1439 VEC_DATA_TYPE(DATA_TYPE, N) \
1440 BASENAME_DST##C = CONVERT(BASENAME_SRC##C, VEC_DATA_TYPE(DATA_TYPE, N));
1441
1442#define CONVERT_ROW_14(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1443 CONVERT_ROW_13(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1444 VEC_DATA_TYPE(DATA_TYPE, N) \
1445 BASENAME_DST##D = CONVERT(BASENAME_SRC##D, VEC_DATA_TYPE(DATA_TYPE, N));
1446
1447#define CONVERT_ROW_15(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1448 CONVERT_ROW_14(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1449 VEC_DATA_TYPE(DATA_TYPE, N) \
1450 BASENAME_DST##E = CONVERT(BASENAME_SRC##E, VEC_DATA_TYPE(DATA_TYPE, N));
1451
1452#define CONVERT_ROW_16(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1453 CONVERT_ROW_15(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1454 VEC_DATA_TYPE(DATA_TYPE, N) \
1455 BASENAME_DST##F = CONVERT(BASENAME_SRC##F, VEC_DATA_TYPE(DATA_TYPE, N));
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1456/** @} */ // end of group CONVERT_ROW_n
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1457
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1458/** Apply convert_<data_type> to a block (BASENAME_SRC) and save to another block (BASENAME_DST)
1459 * @name CONVERT_BLOCK
1460 *
1461 * Supported cases N=1,2,3,...,16.
1462 *
1463 * @param[in] M The number of vectors to convert
1464 * @param[in] N The size of the vectors
1465 * @param[in] DATA_TYPE The data type of the vectors
1466 * @param[in] BASENAME_SRC The basename of the source variables
1467 * @param[in] BASENAME_DST The basename of the destination variables
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1468 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1469#define CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_ROW_##M(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST)
1470#define CONVERT_BLOCK(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]1471/** @} */ // end of group CONVERT_BLOCK