Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / 530488472ae961b53520e7cfe4eb12f35163d93e / . / src / core / CL / cl_kernels / gemm_helpers.h
blob: 3943a8de788f008d075e282b1c2768aaabada20f [file] [log] [blame]
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1/*
Michele Di Giorgiod9eaf612020-07-08 11:12:57 +0100[diff] [blame]2 * Copyright (c) 2019-2020 Arm Limited.
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]24#include "activation_float_helpers.h"
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]25#include "helpers.h"
26
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]27/** Utility macro to access a vector with the scalar positions
28 *
29 * Supported cases are: Offset can only be of the same size of the OpenCL vector (2,3,4,8,16)
30 *
31 * @param[in] offset The offset within the vector. Offset can only be of the same size of the OpenCL vector (2,3,4,8,16)
32 * @param[in] n0 The number of consecutive columns to access. n0 + offset must be <= 16
33 * @param[in] x Vector to access
34 * @{
35 */
36#define SCALAR_ACCESS_STR(offset, n0, x) scalar_access_##offset##_##n0(x)
37#define SCALAR_ACCESS(offset, n0, x) SCALAR_ACCESS_STR(offset, n0, x)
38
39// offset == 0
40#define scalar_access_0_1(x) ((x).s0)
41#define scalar_access_0_2(x) ((x).s01)
42#define scalar_access_0_3(x) ((x).s012)
43#define scalar_access_0_4(x) ((x).s0123)
44#define scalar_access_0_8(x) ((x).s01234567)
45#define scalar_access_0_16(x) ((x).s0123456789ABCDEF)
46
47// offset == 1
48#define scalar_access_1_1(x) ((x).s1)
49#define scalar_access_1_2(x) ((x).s12)
50#define scalar_access_1_3(x) ((x).s123)
51#define scalar_access_1_4(x) ((x).s1234)
52#define scalar_access_1_8(x) ((x).s12345678)
53
54// offset == 2
55#define scalar_access_2_1(x) ((x).s2)
56#define scalar_access_2_2(x) ((x).s23)
57#define scalar_access_2_3(x) ((x).s234)
58#define scalar_access_2_4(x) ((x).s2345)
59#define scalar_access_2_8(x) ((x).s23456789)
60
61// offset == 3
62#define scalar_access_3_1(x) ((x).s3)
63#define scalar_access_3_2(x) ((x).s34)
64#define scalar_access_3_3(x) ((x).s345)
65#define scalar_access_3_4(x) ((x).s3456)
66#define scalar_access_3_8(x) ((x).s3456789A)
67
68// offset == 4
69#define scalar_access_4_1(x) ((x).s4)
70#define scalar_access_4_2(x) ((x).s45)
71#define scalar_access_4_3(x) ((x).s456)
72#define scalar_access_4_4(x) ((x).s4567)
73#define scalar_access_4_8(x) ((x).s456789AB)
74
75// offset == 8
76#define scalar_access_8_1(x) ((x).s8)
77#define scalar_access_8_2(x) ((x).s89)
78#define scalar_access_8_3(x) ((x).s89A)
79#define scalar_access_8_4(x) ((x).s89AB)
80#define scalar_access_8_8(x) ((x).s89ABCDEF)
81
82// offset == 12
83#define scalar_access_12_1(x) ((x).sC)
84#define scalar_access_12_2(x) ((x).sCD)
85#define scalar_access_12_3(x) ((x).sCDE)
86#define scalar_access_12_4(x) ((x).sCDEF)
87
88// offset == 16
89#define scalar_access_16_1(x) ((x).sF)
90
91/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1) without allocating variables.
92 * @name LOAD_TENSOR_ROW_n
93 *
94 * @param[in] N0 The number of columns to load
95 * @param[in] DATA_TYPE The data type of variables
96 * @param[in] BASENAME The basename of the destination variables for the loaded rows
97 * @param[in] PTR The base pointer
98 * @param[in] COL_OFFSET The column vector offset. COL_OFFSET + N0 must be <= 16
99 * @param[in] STRIDE_Y The stride value in y-axis direction
100 * @param[in] Z The z-axis offset vector
101 * @{
102 */
103#define LOAD_TENSOR_ROW_0(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
104 ({})
105
106#define LOAD_TENSOR_ROW_1(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
107 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##0) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
108
109#define LOAD_TENSOR_ROW_2(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
110 LOAD_TENSOR_ROW_1(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
111 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##1) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
112
113#define LOAD_TENSOR_ROW_3(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
114 LOAD_TENSOR_ROW_2(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
115 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##2) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
116
117#define LOAD_TENSOR_ROW_4(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
118 LOAD_TENSOR_ROW_3(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
119 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##3) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
120
121#define LOAD_TENSOR_ROW_5(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
122 LOAD_TENSOR_ROW_4(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
123 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##4) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
124
125#define LOAD_TENSOR_ROW_6(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
126 LOAD_TENSOR_ROW_5(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
127 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##5) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
128
129#define LOAD_TENSOR_ROW_7(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
130 LOAD_TENSOR_ROW_6(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
131 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##6) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
132
133#define LOAD_TENSOR_ROW_8(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
134 LOAD_TENSOR_ROW_7(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
135 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##7) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
136
137#define LOAD_TENSOR_ROW_9(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
138 LOAD_TENSOR_ROW_8(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
139 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##8) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
140
141#define LOAD_TENSOR_ROW_10(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
142 LOAD_TENSOR_ROW_9(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
143 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##9) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
144
145#define LOAD_TENSOR_ROW_11(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
146 LOAD_TENSOR_ROW_10(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
147 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##A) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
148
149#define LOAD_TENSOR_ROW_12(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
150 LOAD_TENSOR_ROW_11(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
151 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##B) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
152
153#define LOAD_TENSOR_ROW_13(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
154 LOAD_TENSOR_ROW_12(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
155 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##C) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
156
157#define LOAD_TENSOR_ROW_14(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
158 LOAD_TENSOR_ROW_13(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
159 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##D) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
160
161#define LOAD_TENSOR_ROW_15(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
162 LOAD_TENSOR_ROW_14(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
163 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##E) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
164
165#define LOAD_TENSOR_ROW_16(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
166 LOAD_TENSOR_ROW_15(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) \
167 SCALAR_ACCESS(COL_OFFSET, N0, BASENAME##F) = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
168/** @}*/ // end of group LOAD_TENSOR_ROW_n
169
170/** Load tensor (consecutive rows and columns) with Z offset.
171 * @name LOAD_TENSOR
172 *
173 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
174 * The data to load is expected to have consecutive names for each row.
175 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
176 * The Z offset is expected to have consecutive names.
177 * E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
178 *
179 * @param[in] M0 The number of consecutive rows
180 * @param[in] N0 The number of consecutive columns
181 * @param[in] DATA_TYPE The data type of the target
182 * @param[in] BASENAME The basename of the result variables
183 * @param[in] PTR The base pointer for the data
184 * @param[in] COL_OFFSET The column vector offset. COL_OFFSET + N0 must be <= 16
185 * @param[in] STRIDE_Y The stride in y-axis direction
186 * @param[in] Z The z-axis offset vector
187 * @{
188 */
189#define LOAD_TENSOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) LOAD_TENSOR_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z)
190#define LOAD_TENSOR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z) LOAD_TENSOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, COL_OFFSET, STRIDE_Y, Z)
191/** @} */ // end of group LOAD_TENSOR
192
193/** Load 2D tensor (consecutive rows and columns) with Z offset.
194 * @name LOAD_TENSOR_M0Xn
195 *
196 * @param[in] M0 The number of rows to load [0-16]
197 * @param[in] N0 The number of columns to load [0-16]
198 * @param[in] DATA_TYPE The data type of variables
199 * @param[in] BASENAME The basename of the destination variables for the loaded rows
200 * @param[in] PTR The base pointer
201 * @param[in] STRIDE_Y The stride value in y-axis direction
202 * @param[in] Z The z-axis offset vector
203 * @{
204 */
205#define LOAD_TENSOR_M0X0(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
206 ({})
207
208#define LOAD_TENSOR_M0X1(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
209 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
210
211#define LOAD_TENSOR_M0X2(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
212 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
213
214#define LOAD_TENSOR_M0X3(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
215 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
216
217#define LOAD_TENSOR_M0X4(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
218 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
219
220#define LOAD_TENSOR_M0X5(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
221 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
222 LOAD_TENSOR(M0, 1, DATA_TYPE, a, input_ptr + 4 * sizeof(DATA_TYPE), 4, src_stride_y, zin);
223
224#define LOAD_TENSOR_M0X6(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
225 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
226 LOAD_TENSOR(M0, 2, DATA_TYPE, a, input_ptr + 4 * sizeof(DATA_TYPE), 4, src_stride_y, zin);
227
228#define LOAD_TENSOR_M0X7(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
229 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
230 LOAD_TENSOR(M0, 3, DATA_TYPE, a, input_ptr + 4 * sizeof(DATA_TYPE), 4, src_stride_y, zin);
231
232#define LOAD_TENSOR_M0X8(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
233 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
234
235#define LOAD_TENSOR_M0X9(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
236 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr 0, src_stride_y, zin); \
237 LOAD_TENSOR(M0, 1, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
238
239#define LOAD_TENSOR_M0X10(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
240 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
241 LOAD_TENSOR(M0, 2, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
242
243#define LOAD_TENSOR_M0X11(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
244 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
245 LOAD_TENSOR(M0, 3, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
246
247#define LOAD_TENSOR_M0X12(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
248 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
249 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin);
250
251#define LOAD_TENSOR_M0X13(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
252 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
253 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin); \
254 LOAD_TENSOR(M0, 1, DATA_TYPE, a, input_ptr + 12 * sizeof(DATA_TYPE), 12, src_stride_y, zin);
255
256#define LOAD_TENSOR_M0X14(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
257 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr 0, src_stride_y, zin); \
258 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin); \
259 LOAD_TENSOR(M0, 2, DATA_TYPE, a, input_ptr + 12 * sizeof(DATA_TYPE), 12, src_stride_y, zin);
260
261#define LOAD_TENSOR_M0X15(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
262 LOAD_TENSOR(M0, 8, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin); \
263 LOAD_TENSOR(M0, 4, DATA_TYPE, a, input_ptr + 8 * sizeof(DATA_TYPE), 8, src_stride_y, zin); \
264 LOAD_TENSOR(M0, 3, DATA_TYPE, a, input_ptr + 12 * sizeof(DATA_TYPE), 12, src_stride_y, zin);
265
266#define LOAD_TENSOR_M0X16(M0, N0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \
267 LOAD_TENSOR(M0, N0, DATA_TYPE, a, input_ptr, 0, src_stride_y, zin);
268/** @}*/ // end of group LOAD_TENSOR_M0Xn
269
270/** Load 2D tensor (consecutive rows and columns) with Z offset.
271 * @name LOAD_TENSOR_M0XN0
272 *
273 * @param[in] M0 The number of consecutive rows [0-16]
274 * @param[in] N0 The number of consecutive columns [0-16]
275 * @param[in] DATA_TYPE The data type of the target
276 * @param[in] BASENAME The basename of the result variables
277 * @param[in] PTR The base pointer for the data
278 * @param[in] STRIDE_Y The stride in y-axis direction
279 * @param[in] Z The z-axis offset vector
280 * @{
281 */
282#define LOAD_TENSOR_M0XN0_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) LOAD_TENSOR_M0X##N0(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
283#define LOAD_TENSOR_M0XN0(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) LOAD_TENSOR_M0XN0_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
284
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]285/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
286 * @name LOAD_ROW_n
287 *
Gian Marco Iodice73cdaac2020-08-10 21:44:14 +0100[diff] [blame]288 * @param[in] N0 The number of columns to load
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]289 * @param[in] DATA_TYPE The data type of variables
290 * @param[in] BASENAME The basename of the destination variables for the loaded rows
291 * @param[in] PTR The base pointer
292 * @param[in] OFFSET The offset within a row
293 * @param[in] STRIDE_Y The stride value in y-axis direction
294 * @param[in] Z The z-axis offset vector
295 * @{
296 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]297#define LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
298 VEC_DATA_TYPE(DATA_TYPE, N0) \
299 BASENAME##0 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y + Z##0));
300
301#define LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
302 LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
303 VEC_DATA_TYPE(DATA_TYPE, N0) \
304 BASENAME##1 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y + Z##1));
305
306#define LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
307 LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
308 VEC_DATA_TYPE(DATA_TYPE, N0) \
309 BASENAME##2 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y + Z##2));
310
311#define LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
312 LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
313 VEC_DATA_TYPE(DATA_TYPE, N0) \
314 BASENAME##3 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y + Z##3));
315
316#define LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
317 LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
318 VEC_DATA_TYPE(DATA_TYPE, N0) \
319 BASENAME##4 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y + Z##4));
320
321#define LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
322 LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
323 VEC_DATA_TYPE(DATA_TYPE, N0) \
324 BASENAME##5 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y + Z##5));
325
326#define LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
327 LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
328 VEC_DATA_TYPE(DATA_TYPE, N0) \
329 BASENAME##6 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y + Z##6));
330
331#define LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
332 LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
333 VEC_DATA_TYPE(DATA_TYPE, N0) \
334 BASENAME##7 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y + Z##7));
335
336#define LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
337 LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
338 VEC_DATA_TYPE(DATA_TYPE, N0) \
339 BASENAME##8 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y + Z##8));
340
341#define LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
342 LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
343 VEC_DATA_TYPE(DATA_TYPE, N0) \
344 BASENAME##9 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y + Z##9));
345
346#define LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
347 LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
348 VEC_DATA_TYPE(DATA_TYPE, N0) \
349 BASENAME##A = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y + Z##A));
350
351#define LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
352 LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
353 VEC_DATA_TYPE(DATA_TYPE, N0) \
354 BASENAME##B = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y + Z##B));
355
356#define LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
357 LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
358 VEC_DATA_TYPE(DATA_TYPE, N0) \
359 BASENAME##C = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y + Z##C));
360
361#define LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
362 LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
363 VEC_DATA_TYPE(DATA_TYPE, N0) \
364 BASENAME##D = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y + Z##D));
365
366#define LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
367 LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
368 VEC_DATA_TYPE(DATA_TYPE, N0) \
369 BASENAME##E = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y + Z##E));
370
371#define LOAD_ROW_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
372 LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
373 VEC_DATA_TYPE(DATA_TYPE, N0) \
374 BASENAME##F = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y + Z##F));
375
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]376/** @}*/ // end of group LOAD_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]377
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]378/** Load Blocks (consecutive rows and columns) with Z offset.
379 * @name LOAD_BLOCK
380 *
381 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
382 * The data to load is expected to have consecutive names for each row.
383 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
384 * The Z offset is expected to have consecutive names.
385 * E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
386 *
387 * @param[in] M0 The number of consecutive rows
388 * @param[in] N0 The number of consecutive columns
389 * @param[in] DATA_TYPE The data type of the target
390 * @param[in] BASENAME The basename of the result variables
391 * @param[in] PTR The base pointer for the data
392 * @param[in] OFFSET The offset within a row
393 * @param[in] STRIDE_Y The stride in y-axis direction
394 * @param[in] Z The z-axis offset vector
395 * @{
396 */
397#define LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
398#define LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
399/** @} */ // end of group LOAD_BLOCK
400
Gian Marco Iodicee3a849a2020-06-10 17:59:30 +0100[diff] [blame]401/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
402 * @name LOAD_TEXTURE2D_ROW_n
403 *
404 * @param[in] N0 The number of pixels to read
405 * @param[in] DATA_TYPE The data type of variables
406 * @param[in] BASENAME The basename of the destination variables for the loaded rows
407 * @param[in] IMG The 2D OpenCL image object
408 * @param[in] X_COORD The x coordinate for the top-left pixel
409 * @param[in] Y_COORD The y coordinate for the top-left pixel
410 * @param[in] X_STEP_ROW The incremental step row for the x coordinate (in pixels)
411 * @param[in] Y_STEP_ROW The incremental step row for the y coordinate (in pixels)
412 * @{
413 */
414#define LOAD_TEXTURE2D_ROW_1(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
415 BASENAME##0 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 0 * X_STEP_ROW), (Y_COORD + 0 * Y_STEP_ROW))
416
417#define LOAD_TEXTURE2D_ROW_2(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
418 LOAD_TEXTURE2D_ROW_1(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
419 BASENAME##1 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 1 * X_STEP_ROW), (Y_COORD + 1 * Y_STEP_ROW))
420
421#define LOAD_TEXTURE2D_ROW_3(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
422 LOAD_TEXTURE2D_ROW_2(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
423 BASENAME##2 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 2 * X_STEP_ROW), (Y_COORD + 2 * Y_STEP_ROW))
424
425#define LOAD_TEXTURE2D_ROW_4(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
426 LOAD_TEXTURE2D_ROW_3(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
427 BASENAME##3 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 3 * X_STEP_ROW), (Y_COORD + 3 * Y_STEP_ROW))
428
429#define LOAD_TEXTURE2D_ROW_5(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
430 LOAD_TEXTURE2D_ROW_4(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
431 BASENAME##4 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 4 * X_STEP_ROW), (Y_COORD + 4 * Y_STEP_ROW))
432
433#define LOAD_TEXTURE2D_ROW_6(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
434 LOAD_TEXTURE2D_ROW_5(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
435 BASENAME##5 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 5 * X_STEP_ROW), (Y_COORD + 5 * Y_STEP_ROW))
436
437#define LOAD_TEXTURE2D_ROW_7(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
438 LOAD_TEXTURE2D_ROW_6(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
439 BASENAME##6 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 6 * X_STEP_ROW), (Y_COORD + 6 * Y_STEP_ROW))
440
441#define LOAD_TEXTURE2D_ROW_8(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
442 LOAD_TEXTURE2D_ROW_7(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
443 BASENAME##7 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 7 * X_STEP_ROW), (Y_COORD + 7 * Y_STEP_ROW))
444
445#define LOAD_TEXTURE2D_ROW_9(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
446 LOAD_TEXTURE2D_ROW_8(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
447 BASENAME##8 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 8 * X_STEP_ROW), (Y_COORD + 8 * Y_STEP_ROW))
448
449#define LOAD_TEXTURE2D_ROW_10(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
450 LOAD_TEXTURE2D_ROW_9(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
451 BASENAME##9 = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 9 * X_STEP_ROW), (Y_COORD + 9 * Y_STEP_ROW))
452
453#define LOAD_TEXTURE2D_ROW_11(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
454 LOAD_TEXTURE2D_ROW_10(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
455 BASENAME##A = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 10 * X_STEP_ROW), (Y_COORD + 10 * Y_STEP_ROW))
456
457#define LOAD_TEXTURE2D_ROW_12(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
458 LOAD_TEXTURE2D_ROW_11(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
459 BASENAME##B = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 11 * X_STEP_ROW), (Y_COORD + 11 * Y_STEP_ROW))
460
461#define LOAD_TEXTURE2D_ROW_13(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
462 LOAD_TEXTURE2D_ROW_12(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
463 BASENAME##C = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 12 * X_STEP_ROW), (Y_COORD + 12 * Y_STEP_ROW))
464
465#define LOAD_TEXTURE2D_ROW_14(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
466 LOAD_TEXTURE2D_ROW_13(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
467 BASENAME##D = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 13 * X_STEP_ROW), (Y_COORD + 13 * Y_STEP_ROW))
468
469#define LOAD_TEXTURE2D_ROW_15(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
470 LOAD_TEXTURE2D_ROW_14(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
471 BASENAME##E = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 14 * X_STEP_ROW), (Y_COORD + 14 * Y_STEP_ROW))
472
473#define LOAD_TEXTURE2D_ROW_16(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
474 LOAD_TEXTURE2D_ROW_15(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) \
475 BASENAME##F = READ_IMAGE2D(DATA_TYPE, N0, IMG, (X_COORD + 15 * X_STEP_ROW), (Y_COORD + 15 * Y_STEP_ROW))
476/** @} */ // end of group LOAD_TEXTURE2D_ROW_n
477
478/** Load a 2D texture in unit of pixel. A pixel is made of 4 floating point values
479 * @name LOAD_TEXTURE2D
480 *
481 * Supported cases are M0=1,2,3,...,16 and N0=1
482 * The data to load is expected to have consecutive names for each row.
483 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
484 *
485 * @param[in] M0 The number of consecutive rows
486 * @param[in] N0 The number of consecutive pixels. Only 1, 2 and 4 are supported
487 * @param[in] DATA_TYPE The data type of the target
488 * @param[in] BASENAME The basename of the result variables
489 * @param[in] IMG The 2D OpenCL image object
490 * @param[in] X_COORD The x coordinate for the top-left pixel
491 * @param[in] Y_COORD The y coordinate for the top-left pixel
492 * @param[in] X_STEP_ROW The incremental step row for the x coordinate (in pixels)
493 * @param[in] Y_STEP_ROW The incremental step row for the y coordinate (in pixels)
494 * @{
495 */
496#define LOAD_TEXTURE2D_STR(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) LOAD_TEXTURE2D_ROW_##M0(N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW)
497#define LOAD_TEXTURE2D(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW) LOAD_TEXTURE2D_STR(M0, N0, DATA_TYPE, BASENAME, IMG, X_COORD, Y_COORD, X_STEP_ROW, Y_STEP_ROW)
498/** @} */ // end of group LOAD_TEXTURE2D
499
Michele Di Giorgiob54ba282020-01-14 15:31:55 +0000[diff] [blame]500/** Loads the elements from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
501 * @name LOAD_ELEMENT_n
502 *
503 * @param[in] N0 The number of rows to load
504 * @param[in] DATA_TYPE The data type of variables
505 * @param[in] BASENAME The basename of the destination variables for the loaded rows
506 * @param[in] PTR The base pointer
507 * @param[in] OFFSET The offset within a row
508 * @param[in] STRIDE_Y The stride value in y-axis direction
509 * @{
510 */
511#define LOAD_ELEMENT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
512 VEC_DATA_TYPE(DATA_TYPE, N0) \
513 BASENAME##0 = *((__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y));
514
515#define LOAD_ELEMENT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
516 LOAD_ELEMENT_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
517 VEC_DATA_TYPE(DATA_TYPE, N0) \
518 BASENAME##1 = *((__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y));
519
520#define LOAD_ELEMENT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
521 LOAD_ELEMENT_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
522 VEC_DATA_TYPE(DATA_TYPE, N0) \
523 BASENAME##2 = *((__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y));
524
525#define LOAD_ELEMENT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
526 LOAD_ELEMENT_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
527 VEC_DATA_TYPE(DATA_TYPE, N0) \
528 BASENAME##3 = *((__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y));
529
530#define LOAD_ELEMENT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
531 LOAD_ELEMENT_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
532 VEC_DATA_TYPE(DATA_TYPE, N0) \
533 BASENAME##4 = *((__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y));
534
535#define LOAD_ELEMENT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
536 LOAD_ELEMENT_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
537 VEC_DATA_TYPE(DATA_TYPE, N0) \
538 BASENAME##5 = *((__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y));
539
540#define LOAD_ELEMENT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
541 LOAD_ELEMENT_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
542 VEC_DATA_TYPE(DATA_TYPE, N0) \
543 BASENAME##6 = *((__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y));
544
545#define LOAD_ELEMENT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
546 LOAD_ELEMENT_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
547 VEC_DATA_TYPE(DATA_TYPE, N0) \
548 BASENAME##7 = *((__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y));
549
550#define LOAD_ELEMENT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
551 LOAD_ELEMENT_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
552 VEC_DATA_TYPE(DATA_TYPE, N0) \
553 BASENAME##8 = *((__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y));
554
555#define LOAD_ELEMENT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
556 LOAD_ELEMENT_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
557 VEC_DATA_TYPE(DATA_TYPE, N0) \
558 BASENAME##9 = *((__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y));
559
560#define LOAD_ELEMENT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
561 LOAD_ELEMENT_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
562 VEC_DATA_TYPE(DATA_TYPE, N0) \
563 BASENAME##A = *((__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y));
564
565#define LOAD_ELEMENT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
566 LOAD_ELEMENT_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
567 VEC_DATA_TYPE(DATA_TYPE, N0) \
568 BASENAME##B = *((__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y));
569
570#define LOAD_ELEMENT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
571 LOAD_ELEMENT_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
572 VEC_DATA_TYPE(DATA_TYPE, N0) \
573 BASENAME##C = *((__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y));
574
575#define LOAD_ELEMENT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
576 LOAD_ELEMENT_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
577 VEC_DATA_TYPE(DATA_TYPE, N0) \
578 BASENAME##D = *((__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y));
579
580#define LOAD_ELEMENT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
581 LOAD_ELEMENT_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
582 VEC_DATA_TYPE(DATA_TYPE, N0) \
583 BASENAME##E = *((__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y));
584
585#define LOAD_ELEMENT_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
586 LOAD_ELEMENT_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) \
587 VEC_DATA_TYPE(DATA_TYPE, N0) \
588 BASENAME##F = *((__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y));
589
590/** @}*/ // end of group LOAD_ELEMENT_n
591
592/** Load Scalar as Vector (consecutive elements).
593 * @name LOAD_SCALAR_AS_VECTOR
594 *
595 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
596 * The data to load is expected to have consecutive names for each row.
597 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
598 *
599 * @param[in] M0 The number of consecutive rows
600 * @param[in] N0 The number of consecutive columns
601 * @param[in] DATA_TYPE The data type of the target
602 * @param[in] BASENAME The basename of the result variables
603 * @param[in] PTR The base pointer for the data
604 * @param[in] OFFSET The offset within a row
605 * @param[in] STRIDE_Y The stride in y-axis direction
606 * @{
607 */
608#define LOAD_SCALAR_AS_VECTOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) LOAD_ELEMENT_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y)
609#define LOAD_SCALAR_AS_VECTOR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y) LOAD_SCALAR_AS_VECTOR_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y)
610/** @} */ // end of group LOAD_SCALAR_AS_VECTOR
611
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]612/** Basic macros to calculate Z offset values from Z0 to Zn-1
613 * @name CALCULATE_Z_OFFSET_n
614 *
615 * @param[in] M0 The number of offset values to calculate
616 * @param[in] DATA_TYPE The data type of the results
617 * @param[in] Z The basename of the result variables
618 * @param[in] Y The work-itme ID of y-axis
619 * @param[in] HEIGHT_GEMM3D The height of GEMM3D
620 * @param[in] DEPTH_GEMM3D The depth of GEMM3D
621 * @param[in] CROSS_PLANE_PAD The padding required for plane changes accross the z-dimension
622 * @param[in] STRIDE_Y The stride value in y-axis direction
623 *
624 * @{
625 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]626#define CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]627 Z##0 = (0 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
628 Z##0 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##0); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]629 Z##0 *= (CROSS_PLANE_PAD * STRIDE_Y);
630
631#define CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
632 CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]633 Z##1 = (1 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
634 Z##1 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##1); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]635 Z##1 *= (CROSS_PLANE_PAD * STRIDE_Y);
636
637#define CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
638 CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]639 Z##2 = (2 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
640 Z##2 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##2); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]641 Z##2 *= (CROSS_PLANE_PAD * STRIDE_Y);
642
643#define CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
644 CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]645 Z##3 = (3 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
646 Z##3 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##3); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]647 Z##3 *= (CROSS_PLANE_PAD * STRIDE_Y);
648
649#define CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
650 CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]651 Z##4 = (4 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
652 Z##4 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##4); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]653 Z##4 *= (CROSS_PLANE_PAD * STRIDE_Y);
654
655#define CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
656 CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]657 Z##5 = (5 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
658 Z##5 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##5); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]659 Z##5 *= (CROSS_PLANE_PAD * STRIDE_Y);
660
661#define CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
662 CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]663 Z##6 = (6 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
664 Z##6 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##6); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]665 Z##6 *= (CROSS_PLANE_PAD * STRIDE_Y);
666
667#define CALCULATE_Z_OFFSET_8(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
668 CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]669 Z##7 = (7 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
670 Z##7 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##7); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]671 Z##7 *= (CROSS_PLANE_PAD * STRIDE_Y);
672
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]673/** @} */ // end of group CALCULATE_Z_OFFSET_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]674
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]675/** Calculate Z offset values from Z0 to Zn-1
676 * @name CALCULATE_Z_OFFSET
677 *
678 * The Z offsets are expected to have consecutive names.
679 * E.g., for M0=3 and Z=zin, the expected names of Z offsets are zin1, zin2, zin3.
680 * Note that, CROSS_PLANE_PAD (cross plain padding) is required to take into account
681 * the possible cross plane paddings in case of the plance changes across the z-dimension.
682 *
683 * <!--
684 * | |
685 * | plane0 |
686 * | |
687 * |__________________|
688 * |******************|
689 * | cross_plane_pad |
690 * |******************|
691 * | |
692 * | plane1 |
693 * | |
694 * |__________________|
695 * -->
696 *
697 * @param[in] M0 The number of offset values to calculate
698 * @param[in] DATA_TYPE The data type of the results
699 * @param[in] Z The basename of the result variables
700 * @param[in] Y The work-itme ID of y-axis
701 * @param[in] HEIGHT_GEMM3D The height of GEMM3D
702 * @param[in] DEPTH_GEMM3D The depth of GEMM3D
703 * @param[in] CROSS_PLANE_PAD The padding required for plane changes accross the z-dimension
704 * @param[in] STRIDE_Y The stride value in y-axis direction
705 * @{
706 */
707#define CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_##M0(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
708#define CALCULATE_Z_OFFSET(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
709/** @} */ // end of group CALCULATE_Z_OFFSET
710
711/** Store the 0 to (n-1)th rows of the given variables
712 * @name STORE_ROW_n
713 *
SiCong Li3a501662020-06-26 10:02:06 +0100[diff] [blame]714 * @param[in] N0 The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]715 * @param[in] DATA_TYPE The data type of the vectors
716 * @param[in] BASENAME The basename of the variables
717 * @param[in] PTR The base pointer
718 * @param[in] STRIDE_Y The stride value in y-axis direction
719 * @param[in] Z The offset in z-axis direction
720 * @{
721 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]722#define STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
723 VSTORE(N0) \
724 (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
725
726#define STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
727 STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
728 VSTORE(N0) \
729 (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
730
731#define STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
732 STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
733 VSTORE(N0) \
734 (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
735
736#define STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
737 STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
738 VSTORE(N0) \
739 (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
740
741#define STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
742 STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
743 VSTORE(N0) \
744 (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
745
746#define STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
747 STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
748 VSTORE(N0) \
749 (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
750
751#define STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
752 STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
753 VSTORE(N0) \
754 (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
755
756#define STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
757 STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
758 VSTORE(N0) \
759 (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
760
761#define STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
762 STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
763 VSTORE(N0) \
764 (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
765
766#define STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
767 STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
768 VSTORE(N0) \
769 (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
770
771#define STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
772 STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
773 VSTORE(N0) \
774 (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
775
776#define STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
777 STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
778 VSTORE(N0) \
779 (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
780
781#define STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
782 STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
783 VSTORE(N0) \
784 (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
785
786#define STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
787 STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
788 VSTORE(N0) \
789 (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
790
791#define STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
792 STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
793 VSTORE(N0) \
794 (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
795
796#define STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
797 STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
798 VSTORE(N0) \
799 (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]800/** @} */ // end of groupd STORE_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]801
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]802/** Convert and store the 0th to (n-1)th rows of the given variables
803 * @name CONVERT_STORE_ROW_n
804 *
805 * @param[in] N0 The size of the vectors
806 * @param[in] DATA_TYPE The data type of the vectors
807 * @param[in] BASENAME The basename of the variables
808 * @param[in] PTR The base pointer
809 * @param[in] STRIDE_Y The stride value in y-axis direction
810 * @param[in] Z The offset in z-axis direction
811 * @{
812 */
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]813#define CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
814 VSTORE(N0) \
815 (CONVERT_SAT((BASENAME##0), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
816
817#define CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
818 CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
819 VSTORE(N0) \
820 (CONVERT_SAT((BASENAME##1), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
821
822#define CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
823 CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
824 VSTORE(N0) \
825 (CONVERT_SAT((BASENAME##2), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
826
827#define CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
828 CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
829 VSTORE(N0) \
830 (CONVERT_SAT((BASENAME##3), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
831
832#define CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
833 CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
834 VSTORE(N0) \
835 (CONVERT_SAT((BASENAME##4), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
836
837#define CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
838 CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
839 VSTORE(N0) \
840 (CONVERT_SAT((BASENAME##5), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
841
842#define CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
843 CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
844 VSTORE(N0) \
845 (CONVERT_SAT((BASENAME##6), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
846
847#define CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
848 CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
849 VSTORE(N0) \
850 (CONVERT_SAT((BASENAME##7), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
851
852#define CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
853 CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
854 VSTORE(N0) \
855 (CONVERT_SAT((BASENAME##8), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
856
857#define CONVERT_STORE_ROW_10(N0, DATA, BASENAME, PTR, STRIDE_Y, Z) \
858 CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
859 VSTORE(N0) \
860 (CONVERT_SAT((BASENAME##9), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
861
862#define CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
863 CONVERT_STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
864 VSTORE(N0) \
865 (CONVERT_SAT((BASENAME##A), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
866
867#define CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
868 CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
869 VSTORE(N0) \
870 (CONVERT_SAT((BASENAME##B), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
871
872#define CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
873 CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
874 VSTORE(N0) \
875 (CONVERT_SAT((BASENAME##C), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
876
877#define CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
878 CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
879 VSTORE(N0) \
880 (CONVERT_SAT((BASENAME##D), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
881
882#define CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
883 CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
884 VSTORE(N0) \
885 (CONVERT_SAT((BASENAME##E), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
886
887#define CONVERT_STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
888 CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
889 VSTORE(N0) \
890 (CONVERT_SAT((BASENAME##F), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
891
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]892/** @} */ // end of groupd CONVERT_STORE_ROW_n
893
894/** Store a block of the given size M0xN0
895 * @name STORE_BLOCK
896 *
897 * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
898 * The data to store is expected to have consecutive names for each row.
899 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
900 * The Z offset is expected to have consecutive names.
901 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
902 *
903 * @param[in] M0 The number of rows to store
904 * @param[in] N0 The size of each vector
905 * @param[in] DATA_TYPE The data type of the vectors
906 * @param[in] BASENAME The basename of the variables
907 * @param[in] PTR The base pointer
908 * @param[in] STRIDE_Y The stride value in y-axis direction
909 * @param[in] Z The offset in z-axis direction
910 * @{
911 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]912#define STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]913#define STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]914/** @} */ // end of group STORE_BLOCK
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]915
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]916/** Convert and store a block of the given size M0xN0
917 * @name CONVERT_STORE_BLOCK
918 *
919 * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
920 * The data to store is expected to have consecutive names for each row.
921 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
922 * The Z offset is expected to have consecutive names.
923 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
924 *
925 * @param[in] M0 The number of rows to store
926 * @param[in] N0 The size of each vector
927 * @param[in] DATA_TYPE The data type of the vectors
928 * @param[in] BASENAME The basename of the variables
929 * @param[in] PTR The base pointer
930 * @param[in] STRIDE_Y The stride value in y-axis direction
931 * @param[in] Z The offset in z-axis direction
932 * @{
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]933 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]934#define CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]935#define CONVERT_STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]936/** @} */ // end of group CONVERT_STORE_BLOCK
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]937
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]938/** Scale the rows in the given variables (BASENAME0 to BASENAMEn-1)
939 * @name SCALE_ROW_n
940 *
941 * @param[in] DATA_TYPE The data type of the variables
942 * @param[in] BASENAME The basename of the variables
943 * @param[in] SCALE The scale factor
944 * @{
945 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]946#define SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]947 BASENAME##0 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]948
949#define SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
950 SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]951 BASENAME##1 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]952
953#define SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
954 SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]955 BASENAME##2 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]956
957#define SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
958 SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]959 BASENAME##3 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]960
961#define SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
962 SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]963 BASENAME##4 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]964
965#define SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
966 SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]967 BASENAME##5 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]968
969#define SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
970 SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]971 BASENAME##6 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]972
973#define SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
974 SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]975 BASENAME##7 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]976
977#define SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
978 SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]979 BASENAME##8 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]980
981#define SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
982 SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]983 BASENAME##9 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]984
985#define SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
986 SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]987 BASENAME##A *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]988
989#define SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
990 SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]991 BASENAME##B *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]992
993#define SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
994 SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]995 BASENAME##C *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]996
997#define SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
998 SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]999 BASENAME##D *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1000
1001#define SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
1002 SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1003 BASENAME##E *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1004
1005#define SCALE_ROW_16(DATA_TYPE, BASENAME, SCALE) \
1006 SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1007 BASENAME##F *= (DATA_TYPE)SCALE;
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1008/** @} */ // end of group SCALE_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1009
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1010/** Scale elements stored in a block (BASENAME)
1011 * @name SCALE_BLOCK
1012 *
1013 * Supported cases are N=1,2,3,...,16
1014 *
1015 * @param[in] N The number of rows in the block
1016 * @param[in] DATA_TYPE The data type of the block
1017 * @param[in] BASENAME The basename of the block
1018 * @param[in] SCALE The scale factor
1019 * @{
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1020 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1021#define SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE) SCALE_ROW_##N(DATA_TYPE, BASENAME, SCALE)
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1022#define SCALE_BLOCK(N, DATA_TYPE, BASENAME, SCALE) SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1023/** @} */ // end of group SCALE_BLOCK
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1024
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1025/** Create a new vector containing the values at the given index for a set of given vectors
1026 * @name COLUMN_VECTORn
1027 *
1028 * @param[in] IDX_COL The index value
1029 * @param[in] BASENAME The basename of the destination vectors
1030 * @param[in] X The basename of the source vectors
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1031 * @param[in] TYPE The data type of the destination vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1032 * @{
1033 */
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1034#define COLUMN_VECTOR1(IDX_COL, BASENAME, X, TYPE) \
1035 TYPE BASENAME##IDX_COL = (TYPE)((X##0).s##IDX_COL);
1036#define COLUMN_VECTOR2(IDX_COL, BASENAME, X, TYPE) \
1037 VEC_DATA_TYPE(TYPE, 2) \
1038 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 2))((X##0).s##IDX_COL, (X##1).s##IDX_COL);
1039#define COLUMN_VECTOR3(IDX_COL, BASENAME, X, TYPE) \
1040 VEC_DATA_TYPE(TYPE, 3) \
1041 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 3))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL);
1042#define COLUMN_VECTOR4(IDX_COL, BASENAME, X, TYPE) \
1043 VEC_DATA_TYPE(TYPE, 4) \
1044 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 4))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL);
1045#define COLUMN_VECTOR8(IDX_COL, BASENAME, X, TYPE) \
1046 VEC_DATA_TYPE(TYPE, 8) \
1047 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 8))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL, (X##4).s##IDX_COL, (X##5).s##IDX_COL, (X##6).s##IDX_COL, (X##7).s##IDX_COL);
1048#define COLUMN_VECTOR16(IDX_COL, BASENAME, X, TYPE) \
1049 VEC_DATA_TYPE(TYPE, 16) \
1050 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 16))((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL, (X##4).s##IDX_COL, (X##5).s##IDX_COL, (X##6).s##IDX_COL, (X##7).s##IDX_COL, (X##8).s##IDX_COL, (X##9).s##IDX_COL, (X##A).s##IDX_COL, (X##B).s##IDX_COL, (X##C).s##IDX_COL, (X##D).s##IDX_COL, (X##E).s##IDX_COL, (X##F).s##IDX_COL);
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1051/** @} */ // end of group COLUMN_VECTORn
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1052
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1053/** Create a new vector containing the values at the given index. Utility macros for transposing a colum-vector
1054 * @name COLUMN_VECTOR_SCALARn
1055 *
1056 * @param[in] IDX_COL The index value
1057 * @param[in] BASENAME The basename of the destination vectors
1058 * @param[in] X The basename of the source vectors
1059 * @param[in] TYPE The data type of the destination vectors
1060 * @{
1061 */
1062#define COLUMN_VECTOR_SCALAR1(IDX_COL, BASENAME, X, TYPE) \
1063 TYPE BASENAME##IDX_COL = (TYPE)((X##0));
1064#define COLUMN_VECTOR_SCALAR2(IDX_COL, BASENAME, X, TYPE) \
1065 VEC_DATA_TYPE(TYPE, 2) \
1066 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 2))((X##0), (X##1));
1067#define COLUMN_VECTOR_SCALAR3(IDX_COL, BASENAME, X, TYPE) \
1068 VEC_DATA_TYPE(TYPE, 3) \
1069 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 3))((X##0), (X##1), (X##2));
1070#define COLUMN_VECTOR_SCALAR4(IDX_COL, BASENAME, X, TYPE) \
1071 VEC_DATA_TYPE(TYPE, 4) \
1072 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 4))((X##0), (X##1), (X##2), (X##3));
1073#define COLUMN_VECTOR_SCALAR8(IDX_COL, BASENAME, X, TYPE) \
1074 VEC_DATA_TYPE(TYPE, 8) \
1075 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 8))((X##0), (X##1), (X##2), (X##3), (X##4), (X##5), (X##6), (X##7));
1076#define COLUMN_VECTOR_SCALAR16(IDX_COL, BASENAME, X, TYPE) \
1077 VEC_DATA_TYPE(TYPE, 16) \
1078 BASENAME##IDX_COL = (VEC_DATA_TYPE(TYPE, 16))((X##0), (X##1), (X##2), (X##3), (X##4), (X##5), (X##6), (X##7), (X##8), (X##9), (X##A), (X##B), (X##C), (X##D), (X##E), (X##F));
1079/** @} */ // end of group COLUMN_VECTORn
1080
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1081/** Create transposed vectors of the given vectors
1082 * @name TRANSPOSE_K0Xn
1083 *
1084 * @param[in] K0 The size of the source vectors
1085 * @param[in] BASENAME The basename of transposed vectors
1086 * @param[in] B The basename of source vectors for transposition
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1087 * @param[in] TYPE The data type of the transposed vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1088 * @{
1089 */
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1090#define TRANSPOSE_K0X1(K0, BASENAME, B, TYPE) \
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1091 COLUMN_VECTOR_SCALAR(K0, 0, BASENAME, B, TYPE);
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1092#define TRANSPOSE_K0X2(K0, BASENAME, B, TYPE) \
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1093 COLUMN_VECTOR(K0, 0, BASENAME, B, TYPE); \
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1094 COLUMN_VECTOR(K0, 1, BASENAME, B, TYPE);
1095#define TRANSPOSE_K0X3(K0, BASENAME, B, TYPE) \
1096 TRANSPOSE_K0X2(K0, BASENAME, B, TYPE); \
1097 COLUMN_VECTOR(K0, 2, BASENAME, B, TYPE);
1098#define TRANSPOSE_K0X4(K0, BASENAME, B, TYPE) \
1099 TRANSPOSE_K0X3(K0, BASENAME, B, TYPE); \
1100 COLUMN_VECTOR(K0, 3, BASENAME, B, TYPE);
1101#define TRANSPOSE_K0X8(K0, BASENAME, B, TYPE) \
1102 TRANSPOSE_K0X4(K0, BASENAME, B, TYPE); \
1103 COLUMN_VECTOR(K0, 4, BASENAME, B, TYPE); \
1104 COLUMN_VECTOR(K0, 5, BASENAME, B, TYPE); \
1105 COLUMN_VECTOR(K0, 6, BASENAME, B, TYPE); \
1106 COLUMN_VECTOR(K0, 7, BASENAME, B, TYPE);
1107#define TRANSPOSE_K0X16(K0, BASENAME, B, TYPE) \
1108 TRANSPOSE_K0X8(K0, BASENAME, B, TYPE); \
1109 COLUMN_VECTOR(K0, 8, BASENAME, B, TYPE); \
1110 COLUMN_VECTOR(K0, 9, BASENAME, B, TYPE); \
1111 COLUMN_VECTOR(K0, A, BASENAME, B, TYPE); \
1112 COLUMN_VECTOR(K0, B, BASENAME, B, TYPE); \
1113 COLUMN_VECTOR(K0, C, BASENAME, B, TYPE); \
1114 COLUMN_VECTOR(K0, D, BASENAME, B, TYPE); \
1115 COLUMN_VECTOR(K0, E, BASENAME, B, TYPE); \
1116 COLUMN_VECTOR(K0, F, BASENAME, B, TYPE);
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1117
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1118/** @} */ // end of group TRANSPOSE_K0Xn
1119
1120/** Create column vectors to contain the values at the given index for a set of given vectors
1121 *
1122 * @param[in] K0 The number of source vectors
1123 * @param[in] IDX_COL The index value
1124 * @param[in] BASENAME The basename of the destination vectors
1125 * @param[in] B The basename of the source vectors
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1126 * @param[in] TYPE The data type of the destination vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1127 */
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1128#define COLUMN_VECTOR(K0, IDX_COL, BASENAME, B, TYPE) \
1129 CONCAT(COLUMN_VECTOR, K0) \
1130 (IDX_COL, BASENAME, B, TYPE);
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]1131
Gian Marco Iodice061eefd2020-04-23 13:40:00 +0100[diff] [blame]1132/** Create column vectors to contain the values at the given index. Utility macro for transposing a column-vector
1133 *
1134 * @param[in] K0 The number of source vectors
1135 * @param[in] IDX_COL The index value
1136 * @param[in] BASENAME The basename of the destination vectors
1137 * @param[in] B The basename of the source vectors
1138 * @param[in] TYPE The data type of the destination vectors
1139 */
1140#define COLUMN_VECTOR_SCALAR(K0, IDX_COL, BASENAME, B, TYPE) \
1141 CONCAT(COLUMN_VECTOR_SCALAR, K0) \
1142 (IDX_COL, BASENAME, B, TYPE);
1143
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1144/** Create transposed vectors form the given source vectors
1145 *
1146 * @param[in] K0 The size of source vectors
1147 * @param[in] N0 The number of source vectors
1148 * @param[in] BASENAME The basename of transposed vectors
1149 * @param[in] B The basename of source vectors for transposition
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1150 * @param[in] TYPE The data type of the transposed vectors
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1151 *
1152 */
Michele Di Giorgiof9179d32019-11-27 16:17:30 +0000[diff] [blame]1153#define TRANSPOSE_K0XN0(K0, N0, BASENAME, B, TYPE) \
1154 CONCAT(TRANSPOSE_K0X, N0) \
1155 (K0, BASENAME, B, TYPE);
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1156
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1157/** Add the variables (BIAS0 to BIASn-1) to the others (BASENAME0 to BASENAMEn-1)
1158 * @name ADD_ROW_n
1159 *
1160 * @param[in] BASENAME The basename of the destination variables
1161 * @param[in] BIAS The basename of the added variables
1162 * @{
1163 */
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1164#define ADD_ROW_1(BASENAME, BIAS) \
1165 BASENAME##0 += BIAS##0;
1166
1167#define ADD_ROW_2(BASENAME, BIAS) \
1168 ADD_ROW_1(BASENAME, BIAS) \
1169 BASENAME##1 += BIAS##1;
1170
1171#define ADD_ROW_3(BASENAME, BIAS) \
1172 ADD_ROW_2(BASENAME, BIAS) \
1173 BASENAME##2 += BIAS##2;
1174
1175#define ADD_ROW_4(BASENAME, BIAS) \
1176 ADD_ROW_3(BASENAME, BIAS) \
1177 BASENAME##3 += BIAS##3;
1178
1179#define ADD_ROW_5(BASENAME, BIAS) \
1180 ADD_ROW_4(BASENAME, BIAS) \
1181 BASENAME##4 += BIAS##4;
1182
1183#define ADD_ROW_6(BASENAME, BIAS) \
1184 ADD_ROW_5(BASENAME, BIAS) \
1185 BASENAME##5 += BIAS##5;
1186
1187#define ADD_ROW_7(BASENAME, BIAS) \
1188 ADD_ROW_6(BASENAME, BIAS) \
1189 BASENAME##6 += BIAS##6;
1190
1191#define ADD_ROW_8(BASENAME, BIAS) \
1192 ADD_ROW_7(BASENAME, BIAS) \
1193 BASENAME##7 += BIAS##7;
1194
1195#define ADD_ROW_9(BASENAME, BIAS) \
1196 ADD_ROW_8(BASENAME, BIAS) \
1197 BASENAME##8 += BIAS##8;
1198
1199#define ADD_ROW_10(BASENAME, BIAS) \
1200 ADD_ROW_9(BASENAME, BIAS) \
1201 BASENAME##9 += BIAS##9;
1202
1203#define ADD_ROW_11(BASENAME, BIAS) \
1204 ADD_ROW_10(BASENAME, BIAS) \
1205 BASENAME##A += BIAS##A;
1206
1207#define ADD_ROW_12(BASENAME, BIAS) \
1208 ADD_ROW_11(BASENAME, BIAS) \
1209 BASENAME##B += BIAS##B;
1210
1211#define ADD_ROW_13(BASENAME, BIAS) \
1212 ADD_ROW_12(BASENAME, BIAS) \
1213 BASENAME##C += BIAS##C;
1214
1215#define ADD_ROW_14(BASENAME, BIAS) \
1216 ADD_ROW_13(BASENAME, BIAS) \
1217 BASENAME##D += BIAS##D;
1218
1219#define ADD_ROW_15(BASENAME, BIAS) \
1220 ADD_ROW_14(BASENAME, BIAS) \
1221 BASENAME##E += BIAS##E;
1222
1223#define ADD_ROW_16(BASENAME, BIAS) \
1224 ADD_ROW_15(BASENAME, BIAS) \
1225 BASENAME##F += BIAS##F;
1226
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1227/** @} */ // end of group ADD_ROW_n
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1228
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1229/** Add the block (BIAS) to another block (BASENAME)
1230 * @name ADD_BLOCK
1231 *
1232 * Supported cases are N=1,2,3,...,16
1233 *
1234 * @param[in] N The number of vectors in the block
1235 * @param[in] BASENAME The basename of the destination variables
1236 * @param[in] BIAS The basename of the added variables
1237 * @{
1238 */
1239#define ADD_BLOCK_STR(N, BASENAME, BIAS) ADD_ROW_##N(BASENAME, BIAS)
1240#define ADD_BLOCK(N, BASENAME, BIAS) ADD_BLOCK_STR(N, BASENAME, BIAS)
1241/** @} */ // end of group ADD_BLOCK
1242
1243/** Broadcast (add single value) to the each element of the destination variables
1244 * @name ADD_ROW_BROADCAST_n
1245 *
1246 * @param[in] BASENAME The basename of the destination variables
1247 * @param[in] BIAS The variable containing the value to add
1248 * @{
1249 */
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1250#define ADD_ROW_BROADCAST_1(BASENAME, BIAS) \
1251 BASENAME##0 += BIAS;
1252
1253#define ADD_ROW_BROADCAST_2(BASENAME, BIAS) \
1254 ADD_ROW_BROADCAST_1(BASENAME, BIAS) \
1255 BASENAME##1 += BIAS;
1256
1257#define ADD_ROW_BROADCAST_3(BASENAME, BIAS) \
1258 ADD_ROW_BROADCAST_2(BASENAME, BIAS) \
1259 BASENAME##2 += BIAS;
1260
1261#define ADD_ROW_BROADCAST_4(BASENAME, BIAS) \
1262 ADD_ROW_BROADCAST_3(BASENAME, BIAS) \
1263 BASENAME##3 += BIAS;
1264
1265#define ADD_ROW_BROADCAST_5(BASENAME, BIAS) \
1266 ADD_ROW_BROADCAST_4(BASENAME, BIAS) \
1267 BASENAME##4 += BIAS;
1268
1269#define ADD_ROW_BROADCAST_6(BASENAME, BIAS) \
1270 ADD_ROW_BROADCAST_5(BASENAME, BIAS) \
1271 BASENAME##5 += BIAS;
1272
1273#define ADD_ROW_BROADCAST_7(BASENAME, BIAS) \
1274 ADD_ROW_BROADCAST_6(BASENAME, BIAS) \
1275 BASENAME##6 += BIAS;
1276
1277#define ADD_ROW_BROADCAST_8(BASENAME, BIAS) \
1278 ADD_ROW_BROADCAST_7(BASENAME, BIAS) \
1279 BASENAME##7 += BIAS;
1280
1281#define ADD_ROW_BROADCAST_9(BASENAME, BIAS) \
1282 ADD_ROW_BROADCAST_8(BASENAME, BIAS) \
1283 BASENAME##8 += BIAS;
1284
1285#define ADD_ROW_BROADCAST_10(BASENAME, BIAS) \
1286 ADD_ROW_BROADCAST_9(BASENAME, BIAS) \
1287 BASENAME##9 += BIAS;
1288
1289#define ADD_ROW_BROADCAST_11(BASENAME, BIAS) \
1290 ADD_ROW_BROADCAST_10(BASENAME, BIAS) \
1291 BASENAME##A += BIAS;
1292
1293#define ADD_ROW_BROADCAST_12(BASENAME, BIAS) \
1294 ADD_ROW_BROADCAST_11(BASENAME, BIAS) \
1295 BASENAME##B += BIAS;
1296
1297#define ADD_ROW_BROADCAST_13(BASENAME, BIAS) \
1298 ADD_ROW_BROADCAST_12(BASENAME, BIAS) \
1299 BASENAME##C += BIAS;
1300
1301#define ADD_ROW_BROADCAST_14(BASENAME, BIAS) \
1302 ADD_ROW_BROADCAST_13(BASENAME, BIAS) \
1303 BASENAME##D += BIAS;
1304
1305#define ADD_ROW_BROADCAST_15(BASENAME, BIAS) \
1306 ADD_ROW_BROADCAST_14(BASENAME, BIAS) \
1307 BASENAME##E += BIAS;
1308
1309#define ADD_ROW_BROADCAST_16(BASENAME, BIAS) \
1310 ADD_ROW_BROADCAST_15(BASENAME, BIAS) \
1311 BASENAME##F += BIAS;
1312
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1313/** Broadcast (add a value) to the each element of the destination block (BASENAME)
1314 * @name ADD_BLOCK_BROADCAST
1315 *
1316 * Supported cases are N=1,2,3,...,16.
1317 *
1318 * @param[in] N The number of vectors in the block
1319 * @param[in] BASENAME The basename of the destination variables
1320 * @param[in] BIAS The variable containing the value to add
1321 * @{
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1322 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1323#define ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS) ADD_ROW_BROADCAST_##N(BASENAME, BIAS)
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]1324#define ADD_BLOCK_BROADCAST(N, BASENAME, BIAS) ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1325/** @} */ // end of group ADD_BLOCK_BROADCAST
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1326
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1327/** Apply activation to the given variables
1328 * @name ACTIVATION_ROW_n
1329 *
1330 * @param[in] ACTIVATION_TYPE The type of the activation
1331 * @param[in] DATA_TYPE The data type of the vectors
1332 * @param[in] BASENAME The basename of the variables
1333 * @param[in] A_VAL Additional value required by the activation
1334 * @param[in] B_VAL Additional value required by the activation
1335 * @{
1336 */
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1337#define ACTIVATION_ROW_1(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1338 BASENAME##0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##0, A_VAL, B_VAL);
1339
1340#define ACTIVATION_ROW_2(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1341 ACTIVATION_ROW_1(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1342 BASENAME##1 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##1, A_VAL, B_VAL);
1343
1344#define ACTIVATION_ROW_3(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1345 ACTIVATION_ROW_2(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1346 BASENAME##2 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##2, A_VAL, B_VAL);
1347
1348#define ACTIVATION_ROW_4(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1349 ACTIVATION_ROW_3(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1350 BASENAME##3 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##3, A_VAL, B_VAL);
1351
1352#define ACTIVATION_ROW_5(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1353 ACTIVATION_ROW_4(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1354 BASENAME##4 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##4, A_VAL, B_VAL);
1355
1356#define ACTIVATION_ROW_6(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1357 ACTIVATION_ROW_5(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1358 BASENAME##5 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##5, A_VAL, B_VAL);
1359
1360#define ACTIVATION_ROW_7(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1361 ACTIVATION_ROW_6(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1362 BASENAME##6 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##6, A_VAL, B_VAL);
1363
1364#define ACTIVATION_ROW_8(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1365 ACTIVATION_ROW_7(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1366 BASENAME##7 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##7, A_VAL, B_VAL);
1367
1368#define ACTIVATION_ROW_9(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1369 ACTIVATION_ROW_8(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1370 BASENAME##8 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##8, A_VAL, B_VAL);
1371
1372#define ACTIVATION_ROW_10(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1373 ACTIVATION_ROW_9(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1374 BASENAME##9 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##9, A_VAL, B_VAL);
1375
1376#define ACTIVATION_ROW_11(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1377 ACTIVATION_ROW_10(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1378 BASENAME##A = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##A, A_VAL, B_VAL);
1379
1380#define ACTIVATION_ROW_12(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1381 ACTIVATION_ROW_11(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1382 BASENAME##B = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##B, A_VAL, B_VAL);
1383
1384#define ACTIVATION_ROW_13(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1385 ACTIVATION_ROW_12(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1386 BASENAME##C = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##C, A_VAL, B_VAL);
1387
1388#define ACTIVATION_ROW_14(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1389 ACTIVATION_ROW_13(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1390 BASENAME##D = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##D, A_VAL, B_VAL);
1391
1392#define ACTIVATION_ROW_15(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1393 ACTIVATION_ROW_14(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1394 BASENAME##E = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##E, A_VAL, B_VAL);
1395
1396#define ACTIVATION_ROW_16(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1397 ACTIVATION_ROW_15(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
1398 BASENAME##F = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##F, A_VAL, B_VAL);
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1399/** @} */ // end of group ACTIVATION_ROW_n
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1400
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1401/** Apply activation to a block (BASENAME)
1402 * @name ACTIVATION_BLOCK
1403 *
1404 * Supported cases are N=1,2,3,...,16.
1405 *
1406 * @param[in] N The number of vectors in the block
1407 * @param[in] ACTIVATION_TYPE The type of the activation
1408 * @param[in] DATA_TYPE The data type of the vectors
1409 * @param[in] BASENAME The basename of the variables
1410 * @param[in] A_VAL Additional value required by the activation
1411 * @param[in] B_VAL Additional value required by the activation
1412 * @{
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]1413 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1414#define ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) ACTIVATION_ROW_##N(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL)
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1415#define ACTIVATION_BLOCK(N, ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1416/** @} */ // end of group ACTIVATION_BLOCK
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1417
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1418/** Apply convert_<data_type> to the given variables
1419 * @name CONVERT_ROW_n
1420 *
1421 * @param[in] N The size of the vectors
1422 * @param[in] DATA_TYPE The data type of the vectors
1423 * @param[in] BASENAME_SRC The basename of the source variables
1424 * @param[in] BASENAME_DST The basename of the destination variables
1425 */
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1426#define CONVERT_ROW_1(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1427 VEC_DATA_TYPE(DATA_TYPE, N) \
1428 BASENAME_DST##0 = CONVERT(BASENAME_SRC##0, VEC_DATA_TYPE(DATA_TYPE, N));
1429
1430#define CONVERT_ROW_2(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1431 CONVERT_ROW_1(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1432 VEC_DATA_TYPE(DATA_TYPE, N) \
1433 BASENAME_DST##1 = CONVERT(BASENAME_SRC##1, VEC_DATA_TYPE(DATA_TYPE, N));
1434
1435#define CONVERT_ROW_3(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1436 CONVERT_ROW_2(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1437 VEC_DATA_TYPE(DATA_TYPE, N) \
1438 BASENAME_DST##2 = CONVERT(BASENAME_SRC##2, VEC_DATA_TYPE(DATA_TYPE, N));
1439
1440#define CONVERT_ROW_4(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1441 CONVERT_ROW_3(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1442 VEC_DATA_TYPE(DATA_TYPE, N) \
1443 BASENAME_DST##3 = CONVERT(BASENAME_SRC##3, VEC_DATA_TYPE(DATA_TYPE, N));
1444
1445#define CONVERT_ROW_5(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1446 CONVERT_ROW_4(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1447 VEC_DATA_TYPE(DATA_TYPE, N) \
1448 BASENAME_DST##4 = CONVERT(BASENAME_SRC##4, VEC_DATA_TYPE(DATA_TYPE, N));
1449
1450#define CONVERT_ROW_6(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1451 CONVERT_ROW_5(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1452 VEC_DATA_TYPE(DATA_TYPE, N) \
1453 BASENAME_DST##5 = CONVERT(BASENAME_SRC##5, VEC_DATA_TYPE(DATA_TYPE, N));
1454
1455#define CONVERT_ROW_7(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1456 CONVERT_ROW_6(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1457 VEC_DATA_TYPE(DATA_TYPE, N) \
1458 BASENAME_DST##6 = CONVERT(BASENAME_SRC##6, VEC_DATA_TYPE(DATA_TYPE, N));
1459
1460#define CONVERT_ROW_8(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1461 CONVERT_ROW_7(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1462 VEC_DATA_TYPE(DATA_TYPE, N) \
1463 BASENAME_DST##7 = CONVERT(BASENAME_SRC##7, VEC_DATA_TYPE(DATA_TYPE, N));
1464
1465#define CONVERT_ROW_9(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1466 CONVERT_ROW_8(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1467 VEC_DATA_TYPE(DATA_TYPE, N) \
1468 BASENAME_DST##8 = CONVERT(BASENAME_SRC##8, VEC_DATA_TYPE(DATA_TYPE, N));
1469
1470#define CONVERT_ROW_10(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1471 CONVERT_ROW_9(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1472 VEC_DATA_TYPE(DATA_TYPE, N) \
1473 BASENAME_DST##9 = CONVERT(BASENAME_SRC##9, VEC_DATA_TYPE(DATA_TYPE, N));
1474
1475#define CONVERT_ROW_11(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1476 CONVERT_ROW_10(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1477 VEC_DATA_TYPE(DATA_TYPE, N) \
1478 BASENAME_DST##A = CONVERT(BASENAME_SRC##A, VEC_DATA_TYPE(DATA_TYPE, N));
1479
1480#define CONVERT_ROW_12(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1481 CONVERT_ROW_11(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1482 VEC_DATA_TYPE(DATA_TYPE, N) \
1483 BASENAME_DST##B = CONVERT(BASENAME_SRC##B, VEC_DATA_TYPE(DATA_TYPE, N));
1484
1485#define CONVERT_ROW_13(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1486 CONVERT_ROW_12(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1487 VEC_DATA_TYPE(DATA_TYPE, N) \
1488 BASENAME_DST##C = CONVERT(BASENAME_SRC##C, VEC_DATA_TYPE(DATA_TYPE, N));
1489
1490#define CONVERT_ROW_14(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1491 CONVERT_ROW_13(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1492 VEC_DATA_TYPE(DATA_TYPE, N) \
1493 BASENAME_DST##D = CONVERT(BASENAME_SRC##D, VEC_DATA_TYPE(DATA_TYPE, N));
1494
1495#define CONVERT_ROW_15(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1496 CONVERT_ROW_14(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1497 VEC_DATA_TYPE(DATA_TYPE, N) \
1498 BASENAME_DST##E = CONVERT(BASENAME_SRC##E, VEC_DATA_TYPE(DATA_TYPE, N));
1499
1500#define CONVERT_ROW_16(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1501 CONVERT_ROW_15(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
1502 VEC_DATA_TYPE(DATA_TYPE, N) \
1503 BASENAME_DST##F = CONVERT(BASENAME_SRC##F, VEC_DATA_TYPE(DATA_TYPE, N));
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1504/** @} */ // end of group CONVERT_ROW_n
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1505
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1506/** Apply convert_<data_type> to a block (BASENAME_SRC) and save to another block (BASENAME_DST)
1507 * @name CONVERT_BLOCK
1508 *
1509 * Supported cases N=1,2,3,...,16.
1510 *
1511 * @param[in] M The number of vectors to convert
1512 * @param[in] N The size of the vectors
1513 * @param[in] DATA_TYPE The data type of the vectors
1514 * @param[in] BASENAME_SRC The basename of the source variables
1515 * @param[in] BASENAME_DST The basename of the destination variables
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]1516 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]1517#define CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_ROW_##M(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST)
1518#define CONVERT_BLOCK(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST)
Giorgio Arenad304adb2020-10-02 10:20:11 +0100[diff] [blame]1519/** @} */ // end of group CONVERT_BLOCK