Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / ed7b27dd7cbdae57b880029840ad0235523848e0 / . / src / core / CL / cl_kernels / direct_convolution_quantized.cl
blob: 1182428cd5cde2cd3b4f98b6d4b4e5737875a7ac [file] [log] [blame]
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]1/*
Michele Di Giorgioa046e162019-10-08 09:36:26 +0100[diff] [blame]2 * Copyright (c) 2017-2019 ARM Limited.
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "helpers_asymm.h"
25
26#undef CONVERT_SAT
27
28#if defined(DATA_TYPE) && defined(STRIDE_X) && defined(WEIGHTS_DEPTH)
29
Sang-Hoon Parkab5b1a22019-10-15 09:29:13 +0100[diff] [blame]30#if KERNEL_SIZE == 9
31
32#if STRIDE_X == 1
33#define CONVOLUTION1x9(acc, src_row_ptr, weights_row_ptr) CONVOLUTION1x9_STRIDE1(acc, src_row_ptr, weights_row_ptr)
34#elif STRIDE_X == 2
35#define CONVOLUTION1x9(acc, src_row_ptr, weights_row_ptr) CONVOLUTION1x9_STRIDE2(acc, src_row_ptr, weights_row_ptr)
36#else /* STRIDE_X not equals 1 or 2 */
37#error "STRIDE_X larger than 2 is not supported"
38#endif /* STRIDE_X */
39
40#define CONVOLUTION1x9_STRIDE1(acc, src_row_ptr, weights_row_ptr) \
41 ({ \
42 int8 weights_values0 = convert_int8(vload8(0, weights_row_ptr)); \
43 int weights_value1 = convert_int(*(weights_row_ptr + 8)); \
44 int16 src0 = convert_int16(vload16(0, src_row_ptr)); \
45 acc += (src0.lo + input_offset) * ((int8)weights_values0.s0 + weight_offset); \
46 acc += ((int8)(src0.s1234, src0.s5678) + input_offset) * ((int8)weights_values0.s1 + weight_offset); \
47 acc += ((int8)(src0.s2345, src0.s6789) + input_offset) * ((int8)weights_values0.s2 + weight_offset); \
48 acc += ((int8)(src0.s3456, src0.s789A) + input_offset) * ((int8)weights_values0.s3 + weight_offset); \
49 acc += ((int8)(src0.s4567, src0.s89AB) + input_offset) * ((int8)weights_values0.s4 + weight_offset); \
50 acc += ((int8)(src0.s5678, src0.s9ABC) + input_offset) * ((int8)weights_values0.s5 + weight_offset); \
51 acc += ((int8)(src0.s6789, src0.sABCD) + input_offset) * ((int8)weights_values0.s6 + weight_offset); \
52 acc += ((int8)(src0.s789A, src0.sBCDE) + input_offset) * ((int8)weights_values0.s7 + weight_offset); \
53 acc += ((int8)(src0.s89AB, src0.sCDEF) + input_offset) * ((int8)weights_value1 + weight_offset); \
54 })
55
56#define CONVOLUTION1x9_STRIDE2(acc, src_row_ptr, weights_row_ptr) \
57 ({ \
58 int8 weights_values0 = convert_int8(vload8(0, weights_row_ptr)); \
59 int weights_value1 = convert_int(*(weights_row_ptr + 8)); \
60 int16 src0 = convert_int16(vload16(0, src_row_ptr)); \
61 int8 src1 = convert_int8(vload8(0, src_row_ptr + 16)); \
62 acc += (src0.even + input_offset) * ((int8)weights_values0.s0 + weight_offset); \
63 acc += ((int8)(src0.s1357, src0.s9BDF) + input_offset) * ((int8)weights_values0.s1 + weight_offset); \
64 acc += ((int8)(src0.s2468, src0.sACE, src1.s0) + input_offset) * ((int8)weights_values0.s2 + weight_offset); \
65 acc += ((int8)(src0.s3579, src0.sBDF, src1.s1) + input_offset) * ((int8)weights_values0.s3 + weight_offset); \
66 acc += ((int8)(src0.s468A, src0.sCE, src1.s02) + input_offset) * ((int8)weights_values0.s4 + weight_offset); \
67 acc += ((int8)(src0.s579B, src0.sDF, src1.s13) + input_offset) * ((int8)weights_values0.s5 + weight_offset); \
68 acc += ((int8)(src0.s68AC, src0.sE, src1.s024) + input_offset) * ((int8)weights_values0.s6 + weight_offset); \
69 acc += ((int8)(src0.s79BD, src0.sF, src1.s135) + input_offset) * ((int8)weights_values0.s7 + weight_offset); \
70 acc += ((int8)(src0.s8ACE, src1.s0246) + input_offset) * ((int8)weights_value1 + weight_offset); \
71 })
72
73#elif KERNEL_SIZE == 5
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]74
75#if STRIDE_X == 1
76#define CONVOLUTION1x5(acc, src_row_ptr, weights_row_ptr) CONVOLUTION1x5_STRIDE1(acc, src_row_ptr, weights_row_ptr)
77#elif STRIDE_X == 2
78#define CONVOLUTION1x5(acc, src_row_ptr, weights_row_ptr) CONVOLUTION1x5_STRIDE2(acc, src_row_ptr, weights_row_ptr)
79#else /* STRIDE_X not equals 1 or 2 */
80#error "STRIDE_X larger than 2 is not supported"
81#endif /* STRIDE_X */
82
83#define CONVOLUTION1x5_STRIDE1(acc, src_row_ptr, weights_row_ptr) \
84 ({ \
85 int4 weights_values0 = convert_int4(vload4(0, weights_row_ptr)); \
86 int weights_value1 = convert_int(*(weights_row_ptr + 4)); \
87 int8 src0 = convert_int8(vload8(0, src_row_ptr)); \
88 int4 src1 = convert_int4(vload4(0, src_row_ptr + 8)); \
89 acc += (src0 + input_offset) * ((int8)weights_values0.s0 + weight_offset); \
90 acc += ((int8)(src0.s1234, src0.s567, src1.s0) + input_offset) * ((int8)weights_values0.s1 + weight_offset); \
91 acc += ((int8)(src0.s234, src0.s567, src1.s01) + input_offset) * ((int8)weights_values0.s2 + weight_offset); \
92 acc += ((int8)(src0.s345, src0.s67, src1.s012) + input_offset) * ((int8)weights_values0.s3 + weight_offset); \
93 acc += ((int8)(src0.s45, src0.s67, src1.s0123) + input_offset) * ((int8)weights_value1 + weight_offset); \
94 })
95
96#define CONVOLUTION1x5_STRIDE2(acc, src_row_ptr, weights_row_ptr) \
97 ({ \
98 int4 weights_values0 = convert_int4(vload4(0, weights_row_ptr)); \
99 int weights_value1 = convert_int(*(weights_row_ptr + 4)); \
100 int16 src0 = convert_int16(vload16(0, src_row_ptr)); \
101 int4 src1 = convert_int4(vload4(0, src_row_ptr + 16)); \
102 acc += (src0.even + input_offset) * ((int8)weights_values0.s0 + weight_offset); \
103 acc += ((int8)(src0.s1357, src0.s9BDF) + input_offset) * ((int8)weights_values0.s1 + weight_offset); \
104 acc += ((int8)(src0.s2468, src0.sACE, src1.s0) + input_offset) * ((int8)weights_values0.s2 + weight_offset); \
105 acc += ((int8)(src0.s3579, src0.sBDF, src1.s1) + input_offset) * ((int8)weights_values0.s3 + weight_offset); \
106 acc += ((int8)(src0.s468a, src0.sCE, src1.s02) + input_offset) * ((int8)weights_value1 + weight_offset); \
107 })
108
109#elif KERNEL_SIZE == 3
110
111#if STRIDE_X == 1
112#define CONVOLUTION1x3(acc, src_row_ptr, weights_row_ptr) CONVOLUTION1x3_STRIDE1(acc, src_row_ptr, weights_row_ptr)
113#elif STRIDE_X == 2
114#define CONVOLUTION1x3(acc, src_row_ptr, weights_row_ptr) CONVOLUTION1x3_STRIDE2(acc, src_row_ptr, weights_row_ptr)
115#else /* STRIDE_X not equals 1 or 2 */
116#error "STRIDE_X larger than 2 is not supported"
117#endif /* STRIDE_X */
118
119#define CONVOLUTION1x3_STRIDE1(acc, src_row_ptr, weights_row_ptr) \
120 ({ \
121 int3 weights_values0 = convert_int3(vload3(0, weights_row_ptr)); \
122 int8 src0 = convert_int8(vload8(0, src_row_ptr)); \
123 int2 src1 = convert_int2(vload2(0, src_row_ptr + 8)); \
124 acc += (src0 + input_offset) * ((int8)weights_values0.s0 + weight_offset); \
125 acc += ((int8)(src0.s1234, src0.s567, src1.s0) + input_offset) * ((int8)weights_values0.s1 + weight_offset); \
126 acc += ((int8)(src0.s234, src0.s567, src1.s01) + input_offset) * ((int8)weights_values0.s2 + weight_offset); \
127 })
128
129#define CONVOLUTION1x3_STRIDE2(acc, src_row_ptr, weights_row_ptr) \
130 ({ \
131 int3 weights_values0 = convert_int3(vload3(0, weights_row_ptr)); \
132 int16 src0 = convert_int16(vload16(0, src_row_ptr)); \
133 int src1 = convert_int(*(src_row_ptr + 16)); \
134 acc += (src0.even + input_offset) * ((int8)weights_values0.s0 + weight_offset); \
135 acc += ((int8)(src0.s1357, src0.s9BDF) + input_offset) * ((int8)weights_values0.s1 + weight_offset); \
136 acc += ((int8)(src0.s2468, src0.sACE, src1) + input_offset) * ((int8)weights_values0.s2 + weight_offset); \
137 })
138
139#elif KERNEL_SIZE == 1
140
141#if STRIDE_X == 3
142#define INPUT_PIXEL extract_input_stride3
143#elif STRIDE_X == 2
144#define INPUT_PIXEL extract_input_stride2
145#elif STRIDE_X == 1
146#define INPUT_PIXEL extract_input_stride1
147
148#else /* STRIDE_X not equals 1, 2 or 3 */
149#error "Only support strides 1, 2 and 3"
150#endif /* STRIDE_X */
151
152/** Extracts a 1D horizontal vector from the input tensor with stride as 1.
153 *
154 * @param[in] input_pixel Pointer to the first pixel.
155 *
156 * @return extracted input pixels.
157 */
158inline uchar8 extract_input_stride1(__global const uchar *input_pixel)
159{
160 return vload8(0, input_pixel);
161}
162
163/** Extracts a 1D horizontal vector from the input tensor with stride as 2.
164 *
165 * @param[in] input_pixel Pointer to the first pixel.
166 *
167 * @return extracted input pixels.
168 */
169inline uchar8 extract_input_stride2(__global const uchar *input_pixel)
170{
171 uchar16 temp = vload16(0, input_pixel);
172 return temp.s02468ace;
173}
174
175/** Extracts a 1D horizontal vector from the input tensor with stride as 3 and 8-bit data size.
176 *
177 * @param[in] input_pixel Pointer to the first pixel.
178 *
179 * @return extracted input pixels.
180 */
181inline uchar8 extract_input_stride3(__global const uchar *input_pixel)
182{
183 uchar16 temp1 = vload16(0, input_pixel);
184 uchar16 temp2 = vload16(0, input_pixel + 12);
185 return (uchar8)(temp1.s0369, temp2.s0369);
186}
187
Sang-Hoon Parkab5b1a22019-10-15 09:29:13 +0100[diff] [blame]188#else /* KERNEL_SIZE not equals 1, 3 , 5, 9 */
189#error "Only kernel sizes 1, 3, 5 and 9 are supported"
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]190#endif /* KERNEL_SIZE */
191
192/** This kernel performs a direct convolution to convolve the low three dimensions.
193 *
194 * @note The convolution stride x must be passed at compile time using -DSTRIDE_X e.g. -DSTRIDE_X=1
195 * @note The third dimensions of the weights tensors must be passed at compile time using -DWEIGHTS_DEPTH
196 * @note If biases are used then -DHAS_BIAS has to be passed at compile time
197 *
198 * @param[in] src_ptr Pointer to the source tensor. Supported data types: QASYMM8
199 * @param[in] src_stride_x Stride of the source tensor in X dimension (in bytes)
200 * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes)
201 * @param[in] src_stride_y Stride of the source tensor in Y dimension (in bytes)
202 * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes)
203 * @param[in] src_stride_z Stride of the source tensor in Z dimension (in bytes)
204 * @param[in] src_step_z src_stride_z * number of elements along Z processed per workitem(in bytes)
205 * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source tensor
206 * @param[out] dst_ptr Pointer to the destination tensor. Supported data types: same as @p src_ptr
207 * @param[in] dst_stride_x Stride of the destination tensor in X dimension (in bytes)
208 * @param[in] dst_step_x dst_stride_x * number of elements along X processed per workitem(in bytes)
209 * @param[in] dst_stride_y Stride of the destination tensor in Y dimension (in bytes)
210 * @param[in] dst_step_y dst_stride_y * number of elements along Z processed per workitem(in bytes)
211 * @param[in] dst_stride_z Stride of the destination tensor in Z dimension (in bytes)
212 * @param[in] dst_step_z dst_stride_z * number of elements along Z processed per workitem(in bytes)
213 * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination tensor
Joel Liangf1f3ebd2017-11-10 09:59:19 +0800[diff] [blame]214 * @param[in] weights_ptr Pointer to the weights tensor. Supported data types: same as @p weights_ptr
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]215 * @param[in] weights_stride_x Stride of the weights tensor in X dimension (in bytes)
216 * @param[in] weights_step_x weights_stride_x * number of elements along X processed per workitem(in bytes)
217 * @param[in] weights_stride_y Stride of the weights tensor in Y dimension (in bytes)
218 * @param[in] weights_step_y weights_stride_y * number of elements along y processed per workitem(in bytes)
219 * @param[in] weights_stride_z Stride of the weights tensor in Z dimension (in bytes)
220 * @param[in] weights_step_z weights_stride_z * number of elements along Z processed per workitem(in bytes)
221 * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the weights tensor
Georgios Pinitas540d0082017-11-17 10:55:00 +0000[diff] [blame]222 * @param[in] biases_ptr Pointer to the biases tensor. Supported data types: S32
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]223 * @param[in] biases_stride_x Stride of the biases tensor in X dimension (in bytes)
224 * @param[in] biases_step_x biases_stride_x * number of elements along X processed per workitem(in bytes)
225 * @param[in] biases_offset_first_element_in_bytes The offset of the first element in the biases tensor
226 * @param[in] weights_stride_w Stride of the weights tensor in the 4th dimension
227 * @param[in] input_offset Input offset quantization parameter
228 * @param[in] weight_offset Weights offset quantization parameter
229 * @param[in] output_offset Output offset quantization parameter
230 * @param[in] output_multiplier Output integer multiplier quantization parameter
231 * @param[in] output_shift Output integer shift quantization parameter
232 */
Sang-Hoon Parkab5b1a22019-10-15 09:29:13 +0100[diff] [blame]233__kernel void direct_convolution_quantized(
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]234 TENSOR3D_DECLARATION(src),
235 TENSOR3D_DECLARATION(dst),
236 TENSOR3D_DECLARATION(weights),
237#ifdef HAS_BIAS
238 VECTOR_DECLARATION(biases),
239#endif /* defined(HAS_BIAS) */
240 unsigned int weights_stride_w,
241 int input_offset,
242 int weight_offset,
243 int output_offset,
244 int output_multiplier,
245 int output_shift)
246{
247 Image src = CONVERT_TO_IMAGE_STRUCT(src);
248 Tensor3D weights = CONVERT_TO_TENSOR3D_STRUCT_NO_STEP(weights);
249 Tensor3D dst = CONVERT_TO_TENSOR3D_STRUCT(dst);
250
251 int8 pixels0 = 0;
252
253 __global uchar *weights_addr = (__global uchar *)tensor3D_offset(&weights, 0, 0, 0);
254 __global uchar *src_addr = (__global uchar *)offset(&src, 0, 0);
255
256 const int kernel_index = get_global_id(2);
257 weights_addr += kernel_index * weights_stride_w;
258
259 for(volatile int d = 0; d < WEIGHTS_DEPTH; ++d)
260 {
Sang-Hoon Parkab5b1a22019-10-15 09:29:13 +0100[diff] [blame]261#if KERNEL_SIZE == 9
262 CONVOLUTION1x9(pixels0, (__global uchar *)(src_addr + 0 * src_stride_y), (__global uchar *)(weights_addr + 0 * weights_stride_y));
263 CONVOLUTION1x9(pixels0, (__global uchar *)(src_addr + 1 * src_stride_y), (__global uchar *)(weights_addr + 1 * weights_stride_y));
264 CONVOLUTION1x9(pixels0, (__global uchar *)(src_addr + 2 * src_stride_y), (__global uchar *)(weights_addr + 2 * weights_stride_y));
265 CONVOLUTION1x9(pixels0, (__global uchar *)(src_addr + 3 * src_stride_y), (__global uchar *)(weights_addr + 3 * weights_stride_y));
266 CONVOLUTION1x9(pixels0, (__global uchar *)(src_addr + 4 * src_stride_y), (__global uchar *)(weights_addr + 4 * weights_stride_y));
267 CONVOLUTION1x9(pixels0, (__global uchar *)(src_addr + 5 * src_stride_y), (__global uchar *)(weights_addr + 5 * weights_stride_y));
268 CONVOLUTION1x9(pixels0, (__global uchar *)(src_addr + 6 * src_stride_y), (__global uchar *)(weights_addr + 6 * weights_stride_y));
269 CONVOLUTION1x9(pixels0, (__global uchar *)(src_addr + 7 * src_stride_y), (__global uchar *)(weights_addr + 7 * weights_stride_y));
270 CONVOLUTION1x9(pixels0, (__global uchar *)(src_addr + 8 * src_stride_y), (__global uchar *)(weights_addr + 8 * weights_stride_y));
271#elif KERNEL_SIZE == 5
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]272 CONVOLUTION1x5(pixels0, (__global uchar *)src_addr, (__global uchar *)weights_addr);
273 CONVOLUTION1x5(pixels0, (__global uchar *)(src_addr + 1 * src_stride_y), (__global uchar *)(weights_addr + 1 * weights_stride_y));
274 CONVOLUTION1x5(pixels0, (__global uchar *)(src_addr + 2 * src_stride_y), (__global uchar *)(weights_addr + 2 * weights_stride_y));
275 CONVOLUTION1x5(pixels0, (__global uchar *)(src_addr + 3 * src_stride_y), (__global uchar *)(weights_addr + 3 * weights_stride_y));
276 CONVOLUTION1x5(pixels0, (__global uchar *)(src_addr + 4 * src_stride_y), (__global uchar *)(weights_addr + 4 * weights_stride_y));
277#elif KERNEL_SIZE == 3
278 CONVOLUTION1x3(pixels0, (__global uchar *)(src_addr + 0 * src_stride_y), (__global uchar *)(weights_addr + 0 * weights_stride_y));
279 CONVOLUTION1x3(pixels0, (__global uchar *)(src_addr + 1 * src_stride_y), (__global uchar *)(weights_addr + 1 * weights_stride_y));
280 CONVOLUTION1x3(pixels0, (__global uchar *)(src_addr + 2 * src_stride_y), (__global uchar *)(weights_addr + 2 * weights_stride_y));
281#elif KERNEL_SIZE == 1
282 int weight = convert_int(*(__global uchar *)weights_addr);
283 int8 input_pixel = convert_int8(INPUT_PIXEL((__global uchar *)src_addr));
284 pixels0 += (input_pixel + input_offset) * ((int8)weight + weight_offset);
285#endif /* (KERNEL_SIZE == 1) || (KERNEL_SIZE == 3) || (KERNEL_SIZE == 5) */
286
287 src_addr += src_stride_z;
288 weights_addr += weights_stride_z;
289 }
290
291#ifdef HAS_BIAS
Georgios Pinitas540d0082017-11-17 10:55:00 +0000[diff] [blame]292 Vector biases = CONVERT_TO_VECTOR_STRUCT_NO_STEP(biases);
293 __global int *bias_addr = ((__global int *)(vector_offset(&biases, kernel_index)));
294 pixels0 += (int8)(*bias_addr);
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]295#endif /* defined(HAS_BIAS) */
296
297 pixels0 = ASYMM_MULT_BY_QUANT_MULTIPLIER_LESS_THAN_ONE(pixels0, output_multiplier, output_shift, 8);
298 pixels0 = pixels0 + output_offset;
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]299
Georgios Pinitas6fdfaa82017-11-29 14:27:24 +0000[diff] [blame]300 vstore8(convert_uchar8_sat(pixels0), 0, (__global uchar *)dst.ptr);
Chunosovd621bca2017-11-03 17:33:15 +0700[diff] [blame]301}
302#endif // defined(DATA_TYPE) && defined(STRIDE_X) && defined(WEIGHTS_DEPTH)