Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / 43a129e94df41f9ac8bc78b702da5a387ada0494 / . / src / core / CL / cl_kernels / gemm_helpers.h
blob: 2c76992b313f89fe52dadd3ae52d5e1b6003436d [file] [log] [blame]
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]1/*
2 * Copyright (c) 2019 ARM Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "helpers.h"
25
26#define LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
27 VEC_DATA_TYPE(DATA_TYPE, N0) \
28 BASENAME##0 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y + Z##0));
29
30#define LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
31 LOAD_ROW_1(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
32 VEC_DATA_TYPE(DATA_TYPE, N0) \
33 BASENAME##1 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y + Z##1));
34
35#define LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
36 LOAD_ROW_2(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
37 VEC_DATA_TYPE(DATA_TYPE, N0) \
38 BASENAME##2 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y + Z##2));
39
40#define LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
41 LOAD_ROW_3(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
42 VEC_DATA_TYPE(DATA_TYPE, N0) \
43 BASENAME##3 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y + Z##3));
44
45#define LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
46 LOAD_ROW_4(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
47 VEC_DATA_TYPE(DATA_TYPE, N0) \
48 BASENAME##4 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y + Z##4));
49
50#define LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
51 LOAD_ROW_5(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
52 VEC_DATA_TYPE(DATA_TYPE, N0) \
53 BASENAME##5 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y + Z##5));
54
55#define LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
56 LOAD_ROW_6(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
57 VEC_DATA_TYPE(DATA_TYPE, N0) \
58 BASENAME##6 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y + Z##6));
59
60#define LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
61 LOAD_ROW_7(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
62 VEC_DATA_TYPE(DATA_TYPE, N0) \
63 BASENAME##7 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y + Z##7));
64
65#define LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
66 LOAD_ROW_8(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
67 VEC_DATA_TYPE(DATA_TYPE, N0) \
68 BASENAME##8 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y + Z##8));
69
70#define LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
71 LOAD_ROW_9(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
72 VEC_DATA_TYPE(DATA_TYPE, N0) \
73 BASENAME##9 = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y + Z##9));
74
75#define LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
76 LOAD_ROW_10(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
77 VEC_DATA_TYPE(DATA_TYPE, N0) \
78 BASENAME##A = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y + Z##A));
79
80#define LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
81 LOAD_ROW_11(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
82 VEC_DATA_TYPE(DATA_TYPE, N0) \
83 BASENAME##B = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y + Z##B));
84
85#define LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
86 LOAD_ROW_12(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
87 VEC_DATA_TYPE(DATA_TYPE, N0) \
88 BASENAME##C = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y + Z##C));
89
90#define LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
91 LOAD_ROW_13(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
92 VEC_DATA_TYPE(DATA_TYPE, N0) \
93 BASENAME##D = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y + Z##D));
94
95#define LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
96 LOAD_ROW_14(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
97 VEC_DATA_TYPE(DATA_TYPE, N0) \
98 BASENAME##E = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y + Z##E));
99
100#define LOAD_ROW_16(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
101 LOAD_ROW_15(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \
102 VEC_DATA_TYPE(DATA_TYPE, N0) \
103 BASENAME##F = VLOAD(N0)(0, (__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y + Z##F));
104
105// LOAD_ROW_n loads the rows 0..n-1 in variables BASENAME##0 to BASENAME##(n-1)
106#define LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
107/** Load Blocks of M0 consecutive rows and N0 consecutive columns when using Z offset as well
108 * Supported cases M0=1,2,3..16. N0=1,2,3,4,8,16, for variables BASENAME[0..M0]
109 * The data to load is expected to have consecutive names for each row, For e.g. For M0=3, and basename=c, the expected data is c0, c1 and c2.
110 * The Z offset is expected to have consecutive names For e.g. For M0=3, and Z=zin, the expected z offsets are zin0, zin1 and zin2.
111 */
112#define LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z)
113
114#define CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]115 Z##0 = (0 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
116 Z##0 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##0); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]117 Z##0 *= (CROSS_PLANE_PAD * STRIDE_Y);
118
119#define CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
120 CALCULATE_Z_OFFSET_1(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]121 Z##1 = (1 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
122 Z##1 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##1); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]123 Z##1 *= (CROSS_PLANE_PAD * STRIDE_Y);
124
125#define CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
126 CALCULATE_Z_OFFSET_2(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]127 Z##2 = (2 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
128 Z##2 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##2); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]129 Z##2 *= (CROSS_PLANE_PAD * STRIDE_Y);
130
131#define CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
132 CALCULATE_Z_OFFSET_3(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]133 Z##3 = (3 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
134 Z##3 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##3); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]135 Z##3 *= (CROSS_PLANE_PAD * STRIDE_Y);
136
137#define CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
138 CALCULATE_Z_OFFSET_4(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]139 Z##4 = (4 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
140 Z##4 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##4); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]141 Z##4 *= (CROSS_PLANE_PAD * STRIDE_Y);
142
143#define CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
144 CALCULATE_Z_OFFSET_5(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]145 Z##5 = (5 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
146 Z##5 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##5); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]147 Z##5 *= (CROSS_PLANE_PAD * STRIDE_Y);
148
149#define CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
150 CALCULATE_Z_OFFSET_6(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]151 Z##6 = (6 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
152 Z##6 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##6); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]153 Z##6 *= (CROSS_PLANE_PAD * STRIDE_Y);
154
155#define CALCULATE_Z_OFFSET_8(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
156 CALCULATE_Z_OFFSET_7(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) \
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]157 Z##7 = (7 + (DATA_TYPE)(Y * (DATA_TYPE)M0)) / (DATA_TYPE)HEIGHT_GEMM3D; \
158 Z##7 = min((DATA_TYPE)(DEPTH_GEMM3D - 1), Z##7); \
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]159 Z##7 *= (CROSS_PLANE_PAD * STRIDE_Y);
160
161// CALCULATE_Z_OFFSET_n calculates Z for Z##0 to Z##(n-1)
162#define CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_##M0(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
163/** The Z offsets are expected to have consecutive names, For e.g. For M0=3, and Z=zin, the expected Z offsets are zin1, zin2, zin3.
164 * Note for the REINTERPRET_INPUT_AS_3D case
165 * Since we load a 2D input tile from a 3D tensor, we need to check when the plane changes across the z dimension
166 * in order to take into account the presence of possible cross plane paddings
167 *
168 * | |
169 * | plane0 |
170 * | |
171 * |__________________|
172 * |******************|
173 * | cross_plane_pad |
174 * |******************|
175 * | |
176 * | plane1 |
177 * | |
178 * |__________________|
179 */
180#define CALCULATE_Z_OFFSET(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y) CALCULATE_Z_OFFSET_STR(M0, DATA_TYPE, Z, Y, HEIGHT_GEMM3D, DEPTH_GEMM3D, CROSS_PLANE_PAD, STRIDE_Y)
181
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]182// STORE_ROW_n macros
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]183#define STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
184 VSTORE(N0) \
185 (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
186
187#define STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
188 STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
189 VSTORE(N0) \
190 (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
191
192#define STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
193 STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
194 VSTORE(N0) \
195 (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
196
197#define STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
198 STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
199 VSTORE(N0) \
200 (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
201
202#define STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
203 STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
204 VSTORE(N0) \
205 (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
206
207#define STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
208 STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
209 VSTORE(N0) \
210 (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
211
212#define STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
213 STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
214 VSTORE(N0) \
215 (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
216
217#define STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
218 STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
219 VSTORE(N0) \
220 (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
221
222#define STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
223 STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
224 VSTORE(N0) \
225 (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
226
227#define STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
228 STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
229 VSTORE(N0) \
230 (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
231
232#define STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
233 STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
234 VSTORE(N0) \
235 (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
236
237#define STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
238 STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
239 VSTORE(N0) \
240 (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
241
242#define STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
243 STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
244 VSTORE(N0) \
245 (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
246
247#define STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
248 STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
249 VSTORE(N0) \
250 (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
251
252#define STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
253 STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
254 VSTORE(N0) \
255 (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
256
257#define STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
258 STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
259 VSTORE(N0) \
260 (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
261
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]262// CONVERT_STORE_ROW_n macros
263#define CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
264 VSTORE(N0) \
265 (CONVERT_SAT((BASENAME##0), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));
266
267#define CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
268 CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
269 VSTORE(N0) \
270 (CONVERT_SAT((BASENAME##1), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));
271
272#define CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
273 CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
274 VSTORE(N0) \
275 (CONVERT_SAT((BASENAME##2), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));
276
277#define CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
278 CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
279 VSTORE(N0) \
280 (CONVERT_SAT((BASENAME##3), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));
281
282#define CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
283 CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
284 VSTORE(N0) \
285 (CONVERT_SAT((BASENAME##4), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));
286
287#define CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
288 CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
289 VSTORE(N0) \
290 (CONVERT_SAT((BASENAME##5), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));
291
292#define CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
293 CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
294 VSTORE(N0) \
295 (CONVERT_SAT((BASENAME##6), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));
296
297#define CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
298 CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
299 VSTORE(N0) \
300 (CONVERT_SAT((BASENAME##7), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));
301
302#define CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
303 CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
304 VSTORE(N0) \
305 (CONVERT_SAT((BASENAME##8), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));
306
307#define CONVERT_STORE_ROW_10(N0, DATA, BASENAME, PTR, STRIDE_Y, Z) \
308 CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
309 VSTORE(N0) \
310 (CONVERT_SAT((BASENAME##9), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));
311
312#define CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
313 CONVERT_STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
314 VSTORE(N0) \
315 (CONVERT_SAT((BASENAME##A), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));
316
317#define CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
318 CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
319 VSTORE(N0) \
320 (CONVERT_SAT((BASENAME##B), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));
321
322#define CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
323 CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
324 VSTORE(N0) \
325 (CONVERT_SAT((BASENAME##C), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));
326
327#define CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
328 CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
329 VSTORE(N0) \
330 (CONVERT_SAT((BASENAME##D), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));
331
332#define CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
333 CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
334 VSTORE(N0) \
335 (CONVERT_SAT((BASENAME##E), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));
336
337#define CONVERT_STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
338 CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
339 VSTORE(N0) \
340 (CONVERT_SAT((BASENAME##F), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
341
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]342// STORE_ROW_n stores the rows 0..n-1 from variables BASENAME##0 to BASENAME##(n-1)
343#define STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]344
345// CONVERT_STORE_ROW_n converts and stores the rows 0..n-1 from variables BASENAME##0 to BASENAME##(n-1)
346#define CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
347
348/** Store a block of size M0 (rows) x NO (columns).
349 * Supported cases M0=1,2,3..16. N0=2,3,4,8,16, for variables BASENAME[0..M]
350 * The data to store is expected to have consecutive names for each row, For e.g. For M0=3, and basename=c, the expected data is c0, c1 and c2.
351 * The Z offset is expected to have consecutive names For e.g. For M0=3, and Z=zin, the expected z offsets are zin0, zin1 and zin2.
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]352 */
353#define STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
354
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]355/** Convert and store a block of size M0 (rows) x NO (columns).
356 * Supported cases M0=1,2,3..16. N0=2,3,4,8,16, for variables BASENAME[0..M]
357 * The data to store is expected to have consecutive names for each row, For e.g. For M0=3, and basename=c, the expected data is c0, c1 and c2.
358 * The Z offset is expected to have consecutive names For e.g. For M0=3, and Z=zin, the expected z offsets are zin0, zin1 and zin2.
359 */
360#define CONVERT_STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
361
Usama Arif0681e3b2019-04-25 14:28:07 +0100[diff] [blame]362#define SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) \
363 BASENAME##0 = BASENAME##0 * (DATA_TYPE)SCALE;
364
365#define SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
366 SCALE_ROW_1(DATA_TYPE, BASENAME, SCALE) \
367 BASENAME##1 = BASENAME##1 * (DATA_TYPE)SCALE;
368
369#define SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
370 SCALE_ROW_2(DATA_TYPE, BASENAME, SCALE) \
371 BASENAME##2 = BASENAME##2 * (DATA_TYPE)SCALE;
372
373#define SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
374 SCALE_ROW_3(DATA_TYPE, BASENAME, SCALE) \
375 BASENAME##3 = BASENAME##3 * (DATA_TYPE)SCALE;
376
377#define SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
378 SCALE_ROW_4(DATA_TYPE, BASENAME, SCALE) \
379 BASENAME##4 = BASENAME##4 * (DATA_TYPE)SCALE;
380
381#define SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
382 SCALE_ROW_5(DATA_TYPE, BASENAME, SCALE) \
383 BASENAME##5 = BASENAME##5 * (DATA_TYPE)SCALE;
384
385#define SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
386 SCALE_ROW_6(DATA_TYPE, BASENAME, SCALE) \
387 BASENAME##6 = BASENAME##6 * (DATA_TYPE)SCALE;
388
389#define SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
390 SCALE_ROW_7(DATA_TYPE, BASENAME, SCALE) \
391 BASENAME##7 = BASENAME##7 * (DATA_TYPE)SCALE;
392
393#define SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
394 SCALE_ROW_8(DATA_TYPE, BASENAME, SCALE) \
395 BASENAME##8 = BASENAME##8 * (DATA_TYPE)SCALE;
396
397#define SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
398 SCALE_ROW_9(DATA_TYPE, BASENAME, SCALE) \
399 BASENAME##9 = BASENAME##9 * (DATA_TYPE)SCALE;
400
401#define SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
402 SCALE_ROW_10(DATA_TYPE, BASENAME, SCALE) \
403 BASENAME##A = BASENAME##A * (DATA_TYPE)SCALE;
404
405#define SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
406 SCALE_ROW_11(DATA_TYPE, BASENAME, SCALE) \
407 BASENAME##B = BASENAME##B * (DATA_TYPE)SCALE;
408
409#define SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
410 SCALE_ROW_12(DATA_TYPE, BASENAME, SCALE) \
411 BASENAME##C = BASENAME##C * (DATA_TYPE)SCALE;
412
413#define SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
414 SCALE_ROW_13(DATA_TYPE, BASENAME, SCALE) \
415 BASENAME##D = BASENAME##D * (DATA_TYPE)SCALE;
416
417#define SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
418 SCALE_ROW_14(DATA_TYPE, BASENAME, SCALE) \
419 BASENAME##E = BASENAME##E * (DATA_TYPE)SCALE;
420
421#define SCALE_ROW_16(DATA_TYPE, BASENAME, SCALE) \
422 SCALE_ROW_15(DATA_TYPE, BASENAME, SCALE) \
423 BASENAME##F = BASENAME##F * (DATA_TYPE)SCALE;
424
425// SCALE_ROW_n scales the variables BASENAME##0 to BASENAME##(n-1) by SCALE
426#define SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE) SCALE_ROW_##N(DATA_TYPE, BASENAME, SCALE)
427/** Scale elements stored in variables BASENAME##0 to BASENAME##(N-1) by SCALE
428 * Supported cases N=1,2,3..16, for variables BASENAME[0..N]
429 */
430#define SCALE_BLOCK(N, DATA_TYPE, BASENAME, SCALE) SCALE_BLOCK_STR(N, DATA_TYPE, BASENAME, SCALE)
Gian Marco Iodice43a129e2019-05-14 10:14:08 +0100[diff] [blame^]431
432/** Given a set of vectors of size K0, these macros create a new vector to contain the values at index IDX_COL (with IDX_COL < N0) for all input vectors */
433#define COLUMN_VECTOR1(IDX_COL, BASENAME, B) \
434 uchar BASENAME##IDX_COL = (uchar)((B##0).s##IDX_COL);
435#define COLUMN_VECTOR2(IDX_COL, BASENAME, B) \
436 uchar2 BASENAME##IDX_COL = (uchar2)((B##0).s##IDX_COL, (B##1).s##IDX_COL);
437#define COLUMN_VECTOR3(IDX_COL, BASENAME, B) \
438 uchar3 BASENAME##IDX_COL = (uchar3)((B##0).s##IDX_COL, (B##1).s##IDX_COL, (B##2).s##IDX_COL);
439#define COLUMN_VECTOR4(IDX_COL, BASENAME, B) \
440 uchar4 BASENAME##IDX_COL = (uchar4)((B##0).s##IDX_COL, (B##1).s##IDX_COL, (B##2).s##IDX_COL, (B##3).s##IDX_COL);
441#define COLUMN_VECTOR8(IDX_COL, BASENAME, B) \
442 uchar8 BASENAME##IDX_COL = (uchar8)((B##0).s##IDX_COL, (B##1).s##IDX_COL, (B##2).s##IDX_COL, (B##3).s##IDX_COL, (B##4).s##IDX_COL, (B##5).s##IDX_COL, (B##6).s##IDX_COL, (B##7).s##IDX_COL);
443#define COLUMN_VECTOR16(IDX_COL, BASENAME, B) \
444 uchar16 BASENAME##N0 = (uchar16)((B##0).s##IDX_COL, (B##1).s##IDX_COL, (B##2).s##IDX_COL, (B##3).s##IDX_COL, (B##4).s##IDX_COL, (B##5).s##IDX_COL, (B##6).s##IDX_COL, (B##7).s##IDX_COL, (B##8).s##IDX_COL, (B##9).s##IDX_COL, (B##A).s##IDX_COL, (B##B).s##IDX_COL, (B##C).s##IDX_COL, (B##D).s##IDX_COL, (B##E).s##IDX_COL, (B##F).s##IDX_COL);
445
446/** Given N0 vectors of size K0, these macros create K0 vectors of size N0 which are the result of a transposition */
447#define TRANSPOSE_K0X1(K0, BASENAME, B) \
448 COLUMN_VECTOR(K0, 0, BASENAME, B);
449#define TRANSPOSE_K0X2(K0, BASENAME, B) \
450 TRANSPOSE_K0X1(K0, BASENAME, B); \
451 COLUMN_VECTOR(K0, 1, BASENAME, B);
452#define TRANSPOSE_K0X3(K0, BASENAME, B) \
453 TRANSPOSE_K0X2(K0, BASENAME, B); \
454 COLUMN_VECTOR(K0, 2, BASENAME, B);
455#define TRANSPOSE_K0X4(K0, BASENAME, B) \
456 TRANSPOSE_K0X3(K0, BASENAME, B); \
457 COLUMN_VECTOR(K0, 3, BASENAME, B);
458#define TRANSPOSE_K0X8(K0, BASENAME, B) \
459 TRANSPOSE_K0X4(K0, BASENAME, B); \
460 COLUMN_VECTOR(K0, 4, BASENAME, B); \
461 COLUMN_VECTOR(K0, 5, BASENAME, B); \
462 COLUMN_VECTOR(K0, 6, BASENAME, B); \
463 COLUMN_VECTOR(K0, 7, BASENAME, B);
464#define TRANSPOSE_K0X16(K0, BASENAME, B) \
465 TRANSPOSE_K0X8(K0, BASENAME, B); \
466 COLUMN_VECTOR(K0, 8, BASENAME, B); \
467 COLUMN_VECTOR(K0, 9, BASENAME, B); \
468 COLUMN_VECTOR(K0, A, BASENAME, B); \
469 COLUMN_VECTOR(K0, B, BASENAME, B); \
470 COLUMN_VECTOR(K0, C, BASENAME, B); \
471 COLUMN_VECTOR(K0, D, BASENAME, B); \
472 COLUMN_VECTOR(K0, E, BASENAME, B); \
473 COLUMN_VECTOR(K0, F, BASENAME, B);
474
475#define COLUMN_VECTOR(K0, IDX_COL, BASENAME, B) \
476 CONCAT(COLUMN_VECTOR, K0) \
477 (IDX_COL, BASENAME, B);
478
479#define TRANSPOSE_K0XN0(K0, N0, BASENAME, B) \
480 CONCAT(TRANSPOSE_K0X, N0) \
481 (K0, BASENAME, B);