Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / 14cbfb2921990d8bf125231e350e2ac8dcd95a8b / . / src / core / CL / cl_kernels / gemm_helpers.h
blob: 64914259a4eec9ecc78178740219087165f53116 [file] [log] [blame]
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1/*
2 * Copyright (c) 2019 ARM Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]24#include "activation_float_helpers.h"
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]25#include "helpers.h"
26
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]27/** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1).
28 * @name LOAD_ROW_n
29 *
30 * @param[in] N0 The number of rows to load
31 * @param[in] DATA_TYPE The data type of variables
32 * @param[in] BASENAME The basename of the destination variables for the loaded rows
33 * @param[in] PTR The base pointer
34 * @param[in] OFFSET The offset within a row
35 * @param[in] STRIDE_Y The stride value in y-axis direction
36 * @param[in] Z The z-axis offset vector
37 * @{
38 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]39#define LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
40 VEC_DATA_TYPE(DATA_TYPE, N0) \
41 BASENAME##0 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y + Z##0));
42
43#define LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
44 LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
45 VEC_DATA_TYPE(DATA_TYPE, N0) \
46 BASENAME##1 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y + Z##1));
47
48#define LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
49 LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
50 VEC_DATA_TYPE(DATA_TYPE, N0) \
51 BASENAME##2 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y + Z##2));
52
53#define LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
54 LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
55 VEC_DATA_TYPE(DATA_TYPE, N0) \
56 BASENAME##3 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y + Z##3));
57
58#define LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
59 LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
60 VEC_DATA_TYPE(DATA_TYPE, N0) \
61 BASENAME##4 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y + Z##4));
62
63#define LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
64 LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
65 VEC_DATA_TYPE(DATA_TYPE, N0) \
66 BASENAME##5 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y + Z##5));
67
68#define LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
69 LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
70 VEC_DATA_TYPE(DATA_TYPE, N0) \
71 BASENAME##6 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y + Z##6));
72
73#define LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
74 LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
75 VEC_DATA_TYPE(DATA_TYPE, N0) \
76 BASENAME##7 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y + Z##7));
77
78#define LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
79 LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
80 VEC_DATA_TYPE(DATA_TYPE, N0) \
81 BASENAME##8 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y + Z##8));
82
83#define LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
84 LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
85 VEC_DATA_TYPE(DATA_TYPE, N0) \
86 BASENAME##9 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y + Z##9));
87
88#define LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
89 LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
90 VEC_DATA_TYPE(DATA_TYPE, N0) \
91 BASENAME##A = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y + Z##A));
92
93#define LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
94 LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
95 VEC_DATA_TYPE(DATA_TYPE, N0) \
96 BASENAME##B = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y + Z##B));
97
98#define LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
99 LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
100 VEC_DATA_TYPE(DATA_TYPE, N0) \
101 BASENAME##C = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y + Z##C));
102
103#define LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
104 LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
105 VEC_DATA_TYPE(DATA_TYPE, N0) \
106 BASENAME##D = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y + Z##D));
107
108#define LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
109 LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
110 VEC_DATA_TYPE(DATA_TYPE, N0) \
111 BASENAME##E = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y + Z##E));
112
113#define LOAD_ROW_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
114 LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
115 VEC_DATA_TYPE(DATA_TYPE, N0) \
116 BASENAME##F = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y + Z##F));
117
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]118/** @}*/ // end of group LOAD_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]119
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]120/** Load Blocks (consecutive rows and columns) with Z offset.
121 * @name LOAD_BLOCK
122 *
123 * Supported cases are M0=1,2,3,...,16 and N0=1,2,3,4,8,16
124 * The data to load is expected to have consecutive names for each row.
125 * E.g., for M0=3, and BASENAME=c, the expected data is c0, c1 and c2.
126 * The Z offset is expected to have consecutive names.
127 * E.g., for M0=3, and Z=zin, the expected Z offsets are zin0, zin1 and zin2.
128 *
129 * @param[in] M0 The number of consecutive rows
130 * @param[in] N0 The number of consecutive columns
131 * @param[in] DATA_TYPE The data type of the target
132 * @param[in] BASENAME The basename of the result variables
133 * @param[in] PTR The base pointer for the data
134 * @param[in] OFFSET The offset within a row
135 * @param[in] STRIDE_Y The stride in y-axis direction
136 * @param[in] Z The z-axis offset vector
137 * @{
138 */
139#define LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
140#define LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
141/** @} */ // end of group LOAD_BLOCK
142
143/** Basic macros to calculate Z offset values from Z0 to Zn-1
144 * @name CALCULATE_Z_OFFSET_n
145 *
146 * @param[in] M0 The number of offset values to calculate
147 * @param[in] DATA_TYPE The data type of the results
148 * @param[in] Z The basename of the result variables
149 * @param[in] Y The work-itme ID of y-axis
150 * @param[in] HEIGHT_GEMM3D The height of GEMM3D
151 * @param[in] DEPTH_GEMM3D The depth of GEMM3D
152 * @param[in] CROSS_PLANE_PAD The padding required for plane changes accross the z-dimension
153 * @param[in] STRIDE_Y The stride value in y-axis direction
154 *
155 * @{
156 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]157#define CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]158 Z##0 = (0 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
159 Z##0 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##0); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]160 Z##0 *= (CROSS_PLANE_PAD * STRIDE_Y);
161
162#define CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
163 CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]164 Z##1 = (1 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
165 Z##1 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##1); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]166 Z##1 *= (CROSS_PLANE_PAD * STRIDE_Y);
167
168#define CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
169 CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]170 Z##2 = (2 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
171 Z##2 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##2); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]172 Z##2 *= (CROSS_PLANE_PAD * STRIDE_Y);
173
174#define CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
175 CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]176 Z##3 = (3 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
177 Z##3 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##3); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]178 Z##3 *= (CROSS_PLANE_PAD * STRIDE_Y);
179
180#define CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
181 CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]182 Z##4 = (4 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
183 Z##4 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##4); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]184 Z##4 *= (CROSS_PLANE_PAD * STRIDE_Y);
185
186#define CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
187 CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]188 Z##5 = (5 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
189 Z##5 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##5); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]190 Z##5 *= (CROSS_PLANE_PAD * STRIDE_Y);
191
192#define CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
193 CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]194 Z##6 = (6 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
195 Z##6 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##6); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]196 Z##6 *= (CROSS_PLANE_PAD * STRIDE_Y);
197
198#define CALCULATE_Z_OFFSET_8(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
199 CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]200 Z##7 = (7 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
201 Z##7 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##7); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]202 Z##7 *= (CROSS_PLANE_PAD * STRIDE_Y);
203
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]204/** @} */ // end of group CALCULATE_Z_OFFSET_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]205
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]206/** Calculate Z offset values from Z0 to Zn-1
207 * @name CALCULATE_Z_OFFSET
208 *
209 * The Z offsets are expected to have consecutive names.
210 * E.g., for M0=3 and Z=zin, the expected names of Z offsets are zin1, zin2, zin3.
211 * Note that, CROSS_PLANE_PAD (cross plain padding) is required to take into account
212 * the possible cross plane paddings in case of the plance changes across the z-dimension.
213 *
214 * <!--
215 * | |
216 * | plane0 |
217 * | |
218 * |__________________|
219 * |******************|
220 * | cross_plane_pad |
221 * |******************|
222 * | |
223 * | plane1 |
224 * | |
225 * |__________________|
226 * -->
227 *
228 * @param[in] M0 The number of offset values to calculate
229 * @param[in] DATA_TYPE The data type of the results
230 * @param[in] Z The basename of the result variables
231 * @param[in] Y The work-itme ID of y-axis
232 * @param[in] HEIGHT_GEMM3D The height of GEMM3D
233 * @param[in] DEPTH_GEMM3D The depth of GEMM3D
234 * @param[in] CROSS_PLANE_PAD The padding required for plane changes accross the z-dimension
235 * @param[in] STRIDE_Y The stride value in y-axis direction
236 * @{
237 */
238#define CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_##M0(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
239#define CALCULATE_Z_OFFSET(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
240/** @} */ // end of group CALCULATE_Z_OFFSET
241
242/** Store the 0 to (n-1)th rows of the given variables
243 * @name STORE_ROW_n
244 *
245 * @param[in] N0 The size of the vectors
246 * @param[in] DATA_TYPE The data type of the vectors
247 * @param[in] BASENAME The basename of the variables
248 * @param[in] PTR The base pointer
249 * @param[in] STRIDE_Y The stride value in y-axis direction
250 * @param[in] Z The offset in z-axis direction
251 * @{
252 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]253#define STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
254 VSTORE(N0) \
255 (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
256
257#define STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
258 STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
259 VSTORE(N0) \
260 (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
261
262#define STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
263 STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
264 VSTORE(N0) \
265 (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
266
267#define STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
268 STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
269 VSTORE(N0) \
270 (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
271
272#define STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
273 STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
274 VSTORE(N0) \
275 (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
276
277#define STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
278 STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
279 VSTORE(N0) \
280 (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
281
282#define STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
283 STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
284 VSTORE(N0) \
285 (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
286
287#define STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
288 STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
289 VSTORE(N0) \
290 (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
291
292#define STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
293 STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
294 VSTORE(N0) \
295 (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
296
297#define STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
298 STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
299 VSTORE(N0) \
300 (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
301
302#define STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
303 STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
304 VSTORE(N0) \
305 (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
306
307#define STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
308 STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
309 VSTORE(N0) \
310 (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
311
312#define STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
313 STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
314 VSTORE(N0) \
315 (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
316
317#define STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
318 STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
319 VSTORE(N0) \
320 (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
321
322#define STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
323 STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
324 VSTORE(N0) \
325 (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
326
327#define STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
328 STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
329 VSTORE(N0) \
330 (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]331/** @} */ // end of groupd STORE_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]332
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]333/** Convert and store the 0th to (n-1)th rows of the given variables
334 * @name CONVERT_STORE_ROW_n
335 *
336 * @param[in] N0 The size of the vectors
337 * @param[in] DATA_TYPE The data type of the vectors
338 * @param[in] BASENAME The basename of the variables
339 * @param[in] PTR The base pointer
340 * @param[in] STRIDE_Y The stride value in y-axis direction
341 * @param[in] Z The offset in z-axis direction
342 * @{
343 */
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]344#define CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
345 VSTORE(N0) \
346 (CONVERT_SAT((BASENAME##0), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
347
348#define CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
349 CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
350 VSTORE(N0) \
351 (CONVERT_SAT((BASENAME##1), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
352
353#define CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
354 CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
355 VSTORE(N0) \
356 (CONVERT_SAT((BASENAME##2), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
357
358#define CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
359 CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
360 VSTORE(N0) \
361 (CONVERT_SAT((BASENAME##3), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
362
363#define CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
364 CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
365 VSTORE(N0) \
366 (CONVERT_SAT((BASENAME##4), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
367
368#define CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
369 CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
370 VSTORE(N0) \
371 (CONVERT_SAT((BASENAME##5), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
372
373#define CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
374 CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
375 VSTORE(N0) \
376 (CONVERT_SAT((BASENAME##6), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
377
378#define CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
379 CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
380 VSTORE(N0) \
381 (CONVERT_SAT((BASENAME##7), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
382
383#define CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
384 CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
385 VSTORE(N0) \
386 (CONVERT_SAT((BASENAME##8), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
387
388#define CONVERT_STORE_ROW_10(N0, DATA, BASENAME, PTR, STRIDE_Y, Z) \
389 CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
390 VSTORE(N0) \
391 (CONVERT_SAT((BASENAME##9), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
392
393#define CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
394 CONVERT_STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
395 VSTORE(N0) \
396 (CONVERT_SAT((BASENAME##A), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
397
398#define CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
399 CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
400 VSTORE(N0) \
401 (CONVERT_SAT((BASENAME##B), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
402
403#define CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
404 CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
405 VSTORE(N0) \
406 (CONVERT_SAT((BASENAME##C), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
407
408#define CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
409 CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
410 VSTORE(N0) \
411 (CONVERT_SAT((BASENAME##D), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
412
413#define CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
414 CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
415 VSTORE(N0) \
416 (CONVERT_SAT((BASENAME##E), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
417
418#define CONVERT_STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
419 CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
420 VSTORE(N0) \
421 (CONVERT_SAT((BASENAME##F), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
422
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]423/** @} */ // end of groupd CONVERT_STORE_ROW_n
424
425/** Store a block of the given size M0xN0
426 * @name STORE_BLOCK
427 *
428 * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
429 * The data to store is expected to have consecutive names for each row.
430 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
431 * The Z offset is expected to have consecutive names.
432 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
433 *
434 * @param[in] M0 The number of rows to store
435 * @param[in] N0 The size of each vector
436 * @param[in] DATA_TYPE The data type of the vectors
437 * @param[in] BASENAME The basename of the variables
438 * @param[in] PTR The base pointer
439 * @param[in] STRIDE_Y The stride value in y-axis direction
440 * @param[in] Z The offset in z-axis direction
441 * @{
442 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]443#define STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]444#define STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]445/** @} */ // end of group STORE_BLOCK
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]446
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]447/** Convert and store a block of the given size M0xN0
448 * @name CONVERT_STORE_BLOCK
449 *
450 * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
451 * The data to store is expected to have consecutive names for each row.
452 * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
453 * The Z offset is expected to have consecutive names.
454 * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
455 *
456 * @param[in] M0 The number of rows to store
457 * @param[in] N0 The size of each vector
458 * @param[in] DATA_TYPE The data type of the vectors
459 * @param[in] BASENAME The basename of the variables
460 * @param[in] PTR The base pointer
461 * @param[in] STRIDE_Y The stride value in y-axis direction
462 * @param[in] Z The offset in z-axis direction
463 * @{
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]464 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]465#define CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]466#define CONVERT_STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]467/** @} */ // end of group CONVERT_STORE_BLOCK
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]468
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]469/** Scale the rows in the given variables (BASENAME0 to BASENAMEn-1)
470 * @name SCALE_ROW_n
471 *
472 * @param[in] DATA_TYPE The data type of the variables
473 * @param[in] BASENAME The basename of the variables
474 * @param[in] SCALE The scale factor
475 * @{
476 */
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]477#define SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]478 BASENAME##0 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]479
480#define SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
481 SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]482 BASENAME##1 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]483
484#define SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
485 SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]486 BASENAME##2 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]487
488#define SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
489 SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]490 BASENAME##3 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]491
492#define SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
493 SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]494 BASENAME##4 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]495
496#define SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
497 SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]498 BASENAME##5 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]499
500#define SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
501 SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]502 BASENAME##6 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]503
504#define SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
505 SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]506 BASENAME##7 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]507
508#define SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
509 SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]510 BASENAME##8 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]511
512#define SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
513 SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]514 BASENAME##9 *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]515
516#define SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
517 SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]518 BASENAME##A *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]519
520#define SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
521 SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]522 BASENAME##B *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]523
524#define SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
525 SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]526 BASENAME##C *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]527
528#define SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
529 SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]530 BASENAME##D *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]531
532#define SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
533 SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]534 BASENAME##E *= (DATA_TYPE)SCALE;
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]535
536#define SCALE_ROW_16(DATA_TYPE, BASENAME, SCALE) \
537 SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]538 BASENAME##F *= (DATA_TYPE)SCALE;
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]539/** @} */ // end of group SCALE_ROW_n
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]540
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]541/** Scale elements stored in a block (BASENAME)
542 * @name SCALE_BLOCK
543 *
544 * Supported cases are N=1,2,3,...,16
545 *
546 * @param[in] N The number of rows in the block
547 * @param[in] DATA_TYPE The data type of the block
548 * @param[in] BASENAME The basename of the block
549 * @param[in] SCALE The scale factor
550 * @{
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]551 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]552#define SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE) SCALE_ROW_##N(DATA_TYPE, BASENAME, SCALE)
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]553#define SCALE_BLOCK(N, DATA_TYPE, BASENAME, SCALE) SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]554/** @} */ // end of group SCALE_BLOCK
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]555
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]556/** Create a new vector containing the values at the given index for a set of given vectors
557 * @name COLUMN_VECTORn
558 *
559 * @param[in] IDX_COL The index value
560 * @param[in] BASENAME The basename of the destination vectors
561 * @param[in] X The basename of the source vectors
562 * @{
563 */
Gian Marco Iodicee7510622019-06-03 17:28:17 +0100[diff] [blame]564#define COLUMN_VECTOR1(IDX_COL, BASENAME, X) \
565 uchar BASENAME##IDX_COL = (uchar)((X##0).s##IDX_COL);
566#define COLUMN_VECTOR2(IDX_COL, BASENAME, X) \
567 uchar2 BASENAME##IDX_COL = (uchar2)((X##0).s##IDX_COL, (X##1).s##IDX_COL);
568#define COLUMN_VECTOR3(IDX_COL, BASENAME, X) \
569 uchar3 BASENAME##IDX_COL = (uchar3)((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL);
570#define COLUMN_VECTOR4(IDX_COL, BASENAME, X) \
571 uchar4 BASENAME##IDX_COL = (uchar4)((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL);
572#define COLUMN_VECTOR8(IDX_COL, BASENAME, X) \
573 uchar8 BASENAME##IDX_COL = (uchar8)((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL, (X##4).s##IDX_COL, (X##5).s##IDX_COL, (X##6).s##IDX_COL, (X##7).s##IDX_COL);
574#define COLUMN_VECTOR16(IDX_COL, BASENAME, X) \
575 uchar16 BASENAME##IDX_COL = (uchar16)((X##0).s##IDX_COL, (X##1).s##IDX_COL, (X##2).s##IDX_COL, (X##3).s##IDX_COL, (X##4).s##IDX_COL, (X##5).s##IDX_COL, (X##6).s##IDX_COL, (X##7).s##IDX_COL, (X##8).s##IDX_COL, (X##9).s##IDX_COL, (X##A).s##IDX_COL, (X##B).s##IDX_COL, (X##C).s##IDX_COL, (X##D).s##IDX_COL, (X##E).s##IDX_COL, (X##F).s##IDX_COL);
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]576/** @} */ // end of group COLUMN_VECTORn
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]577
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]578/** Create transposed vectors of the given vectors
579 * @name TRANSPOSE_K0Xn
580 *
581 * @param[in] K0 The size of the source vectors
582 * @param[in] BASENAME The basename of transposed vectors
583 * @param[in] B The basename of source vectors for transposition
584 * @{
585 */
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]586#define TRANSPOSE_K0X1(K0, BASENAME, B) \
587 COLUMN_VECTOR(K0, 0, BASENAME, B);
588#define TRANSPOSE_K0X2(K0, BASENAME, B) \
589 TRANSPOSE_K0X1(K0, BASENAME, B); \
590 COLUMN_VECTOR(K0, 1, BASENAME, B);
591#define TRANSPOSE_K0X3(K0, BASENAME, B) \
592 TRANSPOSE_K0X2(K0, BASENAME, B); \
593 COLUMN_VECTOR(K0, 2, BASENAME, B);
594#define TRANSPOSE_K0X4(K0, BASENAME, B) \
595 TRANSPOSE_K0X3(K0, BASENAME, B); \
596 COLUMN_VECTOR(K0, 3, BASENAME, B);
597#define TRANSPOSE_K0X8(K0, BASENAME, B) \
598 TRANSPOSE_K0X4(K0, BASENAME, B); \
599 COLUMN_VECTOR(K0, 4, BASENAME, B); \
600 COLUMN_VECTOR(K0, 5, BASENAME, B); \
601 COLUMN_VECTOR(K0, 6, BASENAME, B); \
602 COLUMN_VECTOR(K0, 7, BASENAME, B);
603#define TRANSPOSE_K0X16(K0, BASENAME, B) \
604 TRANSPOSE_K0X8(K0, BASENAME, B); \
605 COLUMN_VECTOR(K0, 8, BASENAME, B); \
606 COLUMN_VECTOR(K0, 9, BASENAME, B); \
607 COLUMN_VECTOR(K0, A, BASENAME, B); \
608 COLUMN_VECTOR(K0, B, BASENAME, B); \
609 COLUMN_VECTOR(K0, C, BASENAME, B); \
610 COLUMN_VECTOR(K0, D, BASENAME, B); \
611 COLUMN_VECTOR(K0, E, BASENAME, B); \
612 COLUMN_VECTOR(K0, F, BASENAME, B);
613
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]614/** @} */ // end of group TRANSPOSE_K0Xn
615
616/** Create column vectors to contain the values at the given index for a set of given vectors
617 *
618 * @param[in] K0 The number of source vectors
619 * @param[in] IDX_COL The index value
620 * @param[in] BASENAME The basename of the destination vectors
621 * @param[in] B The basename of the source vectors
622 */
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]623#define COLUMN_VECTOR(K0, IDX_COL, BASENAME, B) \
624 CONCAT(COLUMN_VECTOR, K0) \
625 (IDX_COL, BASENAME, B);
626
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]627/** Create transposed vectors form the given source vectors
628 *
629 * @param[in] K0 The size of source vectors
630 * @param[in] N0 The number of source vectors
631 * @param[in] BASENAME The basename of transposed vectors
632 * @param[in] B The basename of source vectors for transposition
633 *
634 */
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame]635#define TRANSPOSE_K0XN0(K0, N0, BASENAME, B) \
636 CONCAT(TRANSPOSE_K0X, N0) \
637 (K0, BASENAME, B);
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]638
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]639/** Add the variables (BIAS0 to BIASn-1) to the others (BASENAME0 to BASENAMEn-1)
640 * @name ADD_ROW_n
641 *
642 * @param[in] BASENAME The basename of the destination variables
643 * @param[in] BIAS The basename of the added variables
644 * @{
645 */
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]646#define ADD_ROW_1(BASENAME, BIAS) \
647 BASENAME##0 += BIAS##0;
648
649#define ADD_ROW_2(BASENAME, BIAS) \
650 ADD_ROW_1(BASENAME, BIAS) \
651 BASENAME##1 += BIAS##1;
652
653#define ADD_ROW_3(BASENAME, BIAS) \
654 ADD_ROW_2(BASENAME, BIAS) \
655 BASENAME##2 += BIAS##2;
656
657#define ADD_ROW_4(BASENAME, BIAS) \
658 ADD_ROW_3(BASENAME, BIAS) \
659 BASENAME##3 += BIAS##3;
660
661#define ADD_ROW_5(BASENAME, BIAS) \
662 ADD_ROW_4(BASENAME, BIAS) \
663 BASENAME##4 += BIAS##4;
664
665#define ADD_ROW_6(BASENAME, BIAS) \
666 ADD_ROW_5(BASENAME, BIAS) \
667 BASENAME##5 += BIAS##5;
668
669#define ADD_ROW_7(BASENAME, BIAS) \
670 ADD_ROW_6(BASENAME, BIAS) \
671 BASENAME##6 += BIAS##6;
672
673#define ADD_ROW_8(BASENAME, BIAS) \
674 ADD_ROW_7(BASENAME, BIAS) \
675 BASENAME##7 += BIAS##7;
676
677#define ADD_ROW_9(BASENAME, BIAS) \
678 ADD_ROW_8(BASENAME, BIAS) \
679 BASENAME##8 += BIAS##8;
680
681#define ADD_ROW_10(BASENAME, BIAS) \
682 ADD_ROW_9(BASENAME, BIAS) \
683 BASENAME##9 += BIAS##9;
684
685#define ADD_ROW_11(BASENAME, BIAS) \
686 ADD_ROW_10(BASENAME, BIAS) \
687 BASENAME##A += BIAS##A;
688
689#define ADD_ROW_12(BASENAME, BIAS) \
690 ADD_ROW_11(BASENAME, BIAS) \
691 BASENAME##B += BIAS##B;
692
693#define ADD_ROW_13(BASENAME, BIAS) \
694 ADD_ROW_12(BASENAME, BIAS) \
695 BASENAME##C += BIAS##C;
696
697#define ADD_ROW_14(BASENAME, BIAS) \
698 ADD_ROW_13(BASENAME, BIAS) \
699 BASENAME##D += BIAS##D;
700
701#define ADD_ROW_15(BASENAME, BIAS) \
702 ADD_ROW_14(BASENAME, BIAS) \
703 BASENAME##E += BIAS##E;
704
705#define ADD_ROW_16(BASENAME, BIAS) \
706 ADD_ROW_15(BASENAME, BIAS) \
707 BASENAME##F += BIAS##F;
708
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]709/** @} */ // end of group ADD_ROW_n
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]710
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]711/** Add the block (BIAS) to another block (BASENAME)
712 * @name ADD_BLOCK
713 *
714 * Supported cases are N=1,2,3,...,16
715 *
716 * @param[in] N The number of vectors in the block
717 * @param[in] BASENAME The basename of the destination variables
718 * @param[in] BIAS The basename of the added variables
719 * @{
720 */
721#define ADD_BLOCK_STR(N, BASENAME, BIAS) ADD_ROW_##N(BASENAME, BIAS)
722#define ADD_BLOCK(N, BASENAME, BIAS) ADD_BLOCK_STR(N, BASENAME, BIAS)
723/** @} */ // end of group ADD_BLOCK
724
725/** Broadcast (add single value) to the each element of the destination variables
726 * @name ADD_ROW_BROADCAST_n
727 *
728 * @param[in] BASENAME The basename of the destination variables
729 * @param[in] BIAS The variable containing the value to add
730 * @{
731 */
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]732#define ADD_ROW_BROADCAST_1(BASENAME, BIAS) \
733 BASENAME##0 += BIAS;
734
735#define ADD_ROW_BROADCAST_2(BASENAME, BIAS) \
736 ADD_ROW_BROADCAST_1(BASENAME, BIAS) \
737 BASENAME##1 += BIAS;
738
739#define ADD_ROW_BROADCAST_3(BASENAME, BIAS) \
740 ADD_ROW_BROADCAST_2(BASENAME, BIAS) \
741 BASENAME##2 += BIAS;
742
743#define ADD_ROW_BROADCAST_4(BASENAME, BIAS) \
744 ADD_ROW_BROADCAST_3(BASENAME, BIAS) \
745 BASENAME##3 += BIAS;
746
747#define ADD_ROW_BROADCAST_5(BASENAME, BIAS) \
748 ADD_ROW_BROADCAST_4(BASENAME, BIAS) \
749 BASENAME##4 += BIAS;
750
751#define ADD_ROW_BROADCAST_6(BASENAME, BIAS) \
752 ADD_ROW_BROADCAST_5(BASENAME, BIAS) \
753 BASENAME##5 += BIAS;
754
755#define ADD_ROW_BROADCAST_7(BASENAME, BIAS) \
756 ADD_ROW_BROADCAST_6(BASENAME, BIAS) \
757 BASENAME##6 += BIAS;
758
759#define ADD_ROW_BROADCAST_8(BASENAME, BIAS) \
760 ADD_ROW_BROADCAST_7(BASENAME, BIAS) \
761 BASENAME##7 += BIAS;
762
763#define ADD_ROW_BROADCAST_9(BASENAME, BIAS) \
764 ADD_ROW_BROADCAST_8(BASENAME, BIAS) \
765 BASENAME##8 += BIAS;
766
767#define ADD_ROW_BROADCAST_10(BASENAME, BIAS) \
768 ADD_ROW_BROADCAST_9(BASENAME, BIAS) \
769 BASENAME##9 += BIAS;
770
771#define ADD_ROW_BROADCAST_11(BASENAME, BIAS) \
772 ADD_ROW_BROADCAST_10(BASENAME, BIAS) \
773 BASENAME##A += BIAS;
774
775#define ADD_ROW_BROADCAST_12(BASENAME, BIAS) \
776 ADD_ROW_BROADCAST_11(BASENAME, BIAS) \
777 BASENAME##B += BIAS;
778
779#define ADD_ROW_BROADCAST_13(BASENAME, BIAS) \
780 ADD_ROW_BROADCAST_12(BASENAME, BIAS) \
781 BASENAME##C += BIAS;
782
783#define ADD_ROW_BROADCAST_14(BASENAME, BIAS) \
784 ADD_ROW_BROADCAST_13(BASENAME, BIAS) \
785 BASENAME##D += BIAS;
786
787#define ADD_ROW_BROADCAST_15(BASENAME, BIAS) \
788 ADD_ROW_BROADCAST_14(BASENAME, BIAS) \
789 BASENAME##E += BIAS;
790
791#define ADD_ROW_BROADCAST_16(BASENAME, BIAS) \
792 ADD_ROW_BROADCAST_15(BASENAME, BIAS) \
793 BASENAME##F += BIAS;
794
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]795/** Broadcast (add a value) to the each element of the destination block (BASENAME)
796 * @name ADD_BLOCK_BROADCAST
797 *
798 * Supported cases are N=1,2,3,...,16.
799 *
800 * @param[in] N The number of vectors in the block
801 * @param[in] BASENAME The basename of the destination variables
802 * @param[in] BIAS The variable containing the value to add
803 * @{
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]804 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]805#define ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS) ADD_ROW_BROADCAST_##N(BASENAME, BIAS)
Georgios Pinitasb0f342e2019-05-21 13:32:43 +0100[diff] [blame]806#define ADD_BLOCK_BROADCAST(N, BASENAME, BIAS) ADD_BLOCK_BROADCAST_STR(N, BASENAME, BIAS)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]807/** @} */ // end of group ADD_BLOCK_BROADCAST
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]808
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]809/** Apply activation to the given variables
810 * @name ACTIVATION_ROW_n
811 *
812 * @param[in] ACTIVATION_TYPE The type of the activation
813 * @param[in] DATA_TYPE The data type of the vectors
814 * @param[in] BASENAME The basename of the variables
815 * @param[in] A_VAL Additional value required by the activation
816 * @param[in] B_VAL Additional value required by the activation
817 * @{
818 */
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]819#define ACTIVATION_ROW_1(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
820 BASENAME##0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##0, A_VAL, B_VAL);
821
822#define ACTIVATION_ROW_2(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
823 ACTIVATION_ROW_1(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
824 BASENAME##1 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##1, A_VAL, B_VAL);
825
826#define ACTIVATION_ROW_3(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
827 ACTIVATION_ROW_2(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
828 BASENAME##2 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##2, A_VAL, B_VAL);
829
830#define ACTIVATION_ROW_4(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
831 ACTIVATION_ROW_3(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
832 BASENAME##3 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##3, A_VAL, B_VAL);
833
834#define ACTIVATION_ROW_5(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
835 ACTIVATION_ROW_4(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
836 BASENAME##4 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##4, A_VAL, B_VAL);
837
838#define ACTIVATION_ROW_6(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
839 ACTIVATION_ROW_5(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
840 BASENAME##5 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##5, A_VAL, B_VAL);
841
842#define ACTIVATION_ROW_7(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
843 ACTIVATION_ROW_6(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
844 BASENAME##6 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##6, A_VAL, B_VAL);
845
846#define ACTIVATION_ROW_8(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
847 ACTIVATION_ROW_7(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
848 BASENAME##7 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##7, A_VAL, B_VAL);
849
850#define ACTIVATION_ROW_9(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
851 ACTIVATION_ROW_8(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
852 BASENAME##8 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##8, A_VAL, B_VAL);
853
854#define ACTIVATION_ROW_10(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
855 ACTIVATION_ROW_9(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
856 BASENAME##9 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##9, A_VAL, B_VAL);
857
858#define ACTIVATION_ROW_11(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
859 ACTIVATION_ROW_10(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
860 BASENAME##A = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##A, A_VAL, B_VAL);
861
862#define ACTIVATION_ROW_12(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
863 ACTIVATION_ROW_11(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
864 BASENAME##B = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##B, A_VAL, B_VAL);
865
866#define ACTIVATION_ROW_13(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
867 ACTIVATION_ROW_12(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
868 BASENAME##C = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##C, A_VAL, B_VAL);
869
870#define ACTIVATION_ROW_14(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
871 ACTIVATION_ROW_13(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
872 BASENAME##D = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##D, A_VAL, B_VAL);
873
874#define ACTIVATION_ROW_15(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
875 ACTIVATION_ROW_14(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
876 BASENAME##E = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##E, A_VAL, B_VAL);
877
878#define ACTIVATION_ROW_16(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
879 ACTIVATION_ROW_15(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) \
880 BASENAME##F = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, BASENAME##F, A_VAL, B_VAL);
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]881/** @} */ // end of group ACTIVATION_ROW_n
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]882
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]883/** Apply activation to a block (BASENAME)
884 * @name ACTIVATION_BLOCK
885 *
886 * Supported cases are N=1,2,3,...,16.
887 *
888 * @param[in] N The number of vectors in the block
889 * @param[in] ACTIVATION_TYPE The type of the activation
890 * @param[in] DATA_TYPE The data type of the vectors
891 * @param[in] BASENAME The basename of the variables
892 * @param[in] A_VAL Additional value required by the activation
893 * @param[in] B_VAL Additional value required by the activation
894 * @{
Gian Marco Iodiceca1f4602019-07-16 15:46:48 +0100[diff] [blame]895 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]896#define ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) ACTIVATION_ROW_##N(ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL)
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]897#define ACTIVATION_BLOCK(N, ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL) ACTIVATION_BLOCK_STR(N, ACTIVATION_TYPE, DATA_TYPE, BASENAME, A_VAL, B_VAL)
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]898/** @} */ // end of group ACTIVATION_BLOCK
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]899
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]900/** Apply convert_<data_type> to the given variables
901 * @name CONVERT_ROW_n
902 *
903 * @param[in] N The size of the vectors
904 * @param[in] DATA_TYPE The data type of the vectors
905 * @param[in] BASENAME_SRC The basename of the source variables
906 * @param[in] BASENAME_DST The basename of the destination variables
907 */
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]908#define CONVERT_ROW_1(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
909 VEC_DATA_TYPE(DATA_TYPE, N) \
910 BASENAME_DST##0 = CONVERT(BASENAME_SRC##0, VEC_DATA_TYPE(DATA_TYPE, N));
911
912#define CONVERT_ROW_2(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
913 CONVERT_ROW_1(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
914 VEC_DATA_TYPE(DATA_TYPE, N) \
915 BASENAME_DST##1 = CONVERT(BASENAME_SRC##1, VEC_DATA_TYPE(DATA_TYPE, N));
916
917#define CONVERT_ROW_3(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
918 CONVERT_ROW_2(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
919 VEC_DATA_TYPE(DATA_TYPE, N) \
920 BASENAME_DST##2 = CONVERT(BASENAME_SRC##2, VEC_DATA_TYPE(DATA_TYPE, N));
921
922#define CONVERT_ROW_4(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
923 CONVERT_ROW_3(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
924 VEC_DATA_TYPE(DATA_TYPE, N) \
925 BASENAME_DST##3 = CONVERT(BASENAME_SRC##3, VEC_DATA_TYPE(DATA_TYPE, N));
926
927#define CONVERT_ROW_5(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
928 CONVERT_ROW_4(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
929 VEC_DATA_TYPE(DATA_TYPE, N) \
930 BASENAME_DST##4 = CONVERT(BASENAME_SRC##4, VEC_DATA_TYPE(DATA_TYPE, N));
931
932#define CONVERT_ROW_6(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
933 CONVERT_ROW_5(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
934 VEC_DATA_TYPE(DATA_TYPE, N) \
935 BASENAME_DST##5 = CONVERT(BASENAME_SRC##5, VEC_DATA_TYPE(DATA_TYPE, N));
936
937#define CONVERT_ROW_7(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
938 CONVERT_ROW_6(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
939 VEC_DATA_TYPE(DATA_TYPE, N) \
940 BASENAME_DST##6 = CONVERT(BASENAME_SRC##6, VEC_DATA_TYPE(DATA_TYPE, N));
941
942#define CONVERT_ROW_8(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
943 CONVERT_ROW_7(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
944 VEC_DATA_TYPE(DATA_TYPE, N) \
945 BASENAME_DST##7 = CONVERT(BASENAME_SRC##7, VEC_DATA_TYPE(DATA_TYPE, N));
946
947#define CONVERT_ROW_9(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
948 CONVERT_ROW_8(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
949 VEC_DATA_TYPE(DATA_TYPE, N) \
950 BASENAME_DST##8 = CONVERT(BASENAME_SRC##8, VEC_DATA_TYPE(DATA_TYPE, N));
951
952#define CONVERT_ROW_10(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
953 CONVERT_ROW_9(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
954 VEC_DATA_TYPE(DATA_TYPE, N) \
955 BASENAME_DST##9 = CONVERT(BASENAME_SRC##9, VEC_DATA_TYPE(DATA_TYPE, N));
956
957#define CONVERT_ROW_11(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
958 CONVERT_ROW_10(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
959 VEC_DATA_TYPE(DATA_TYPE, N) \
960 BASENAME_DST##A = CONVERT(BASENAME_SRC##A, VEC_DATA_TYPE(DATA_TYPE, N));
961
962#define CONVERT_ROW_12(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
963 CONVERT_ROW_11(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
964 VEC_DATA_TYPE(DATA_TYPE, N) \
965 BASENAME_DST##B = CONVERT(BASENAME_SRC##B, VEC_DATA_TYPE(DATA_TYPE, N));
966
967#define CONVERT_ROW_13(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
968 CONVERT_ROW_12(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
969 VEC_DATA_TYPE(DATA_TYPE, N) \
970 BASENAME_DST##C = CONVERT(BASENAME_SRC##C, VEC_DATA_TYPE(DATA_TYPE, N));
971
972#define CONVERT_ROW_14(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
973 CONVERT_ROW_13(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
974 VEC_DATA_TYPE(DATA_TYPE, N) \
975 BASENAME_DST##D = CONVERT(BASENAME_SRC##D, VEC_DATA_TYPE(DATA_TYPE, N));
976
977#define CONVERT_ROW_15(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
978 CONVERT_ROW_14(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
979 VEC_DATA_TYPE(DATA_TYPE, N) \
980 BASENAME_DST##E = CONVERT(BASENAME_SRC##E, VEC_DATA_TYPE(DATA_TYPE, N));
981
982#define CONVERT_ROW_16(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
983 CONVERT_ROW_15(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) \
984 VEC_DATA_TYPE(DATA_TYPE, N) \
985 BASENAME_DST##F = CONVERT(BASENAME_SRC##F, VEC_DATA_TYPE(DATA_TYPE, N));
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]986/** @} */ // end of group CONVERT_ROW_n
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]987
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]988/** Apply convert_<data_type> to a block (BASENAME_SRC) and save to another block (BASENAME_DST)
989 * @name CONVERT_BLOCK
990 *
991 * Supported cases N=1,2,3,...,16.
992 *
993 * @param[in] M The number of vectors to convert
994 * @param[in] N The size of the vectors
995 * @param[in] DATA_TYPE The data type of the vectors
996 * @param[in] BASENAME_SRC The basename of the source variables
997 * @param[in] BASENAME_DST The basename of the destination variables
Gian Marco Iodice0c17aa22019-09-27 09:23:15 +0100[diff] [blame]998 */
Sang-Hoon Park11b0b8a2019-11-05 13:29:19 +0000[diff] [blame]999#define CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_ROW_##M(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST)
1000#define CONVERT_BLOCK(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST)
1001/** @} */ // end of group CONVERT_BLOCK