blob: 1bf7f2b3ac0e3bc71bd3e5d9a519ad9858fd5c58 [file] [log] [blame]
Georgios Pinitas908f6162021-05-04 10:11:09 +01001/*
Gian Marco Iodice10e88a72021-11-29 12:49:19 +00002 * Copyright (c) 2016-2022 Arm Limited.
Georgios Pinitas908f6162021-05-04 10:11:09 +01003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
Georgios Pinitas7891a732021-08-20 21:39:25 +010024#include "src/gpu/cl/ClKernelLibrary.h"
Georgios Pinitas908f6162021-05-04 10:11:09 +010025
26#include "arm_compute/core/Error.h"
27#include "arm_compute/core/Utils.h"
28
29#include <algorithm>
30#include <array>
31#include <fstream>
32#include <utility>
33
34#ifdef ARM_COMPUTE_COMPRESSED_KERNELS
35#include <zlib.h>
36
37namespace
38{
39/* Decoding table */
40constexpr std::array<uint8_t, 256> b64_invtab =
41{
42 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
43 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
44 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63,
45 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0,
46 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
47 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0,
48 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
49 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0,
50 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
51 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
52 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
53 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
54 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
55 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
56 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
57 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
58};
59
60/** Decode a base64 encoded string
61 *
62 * @param[in] str Base64 encoded string to decode
63 *
64 * @return The decode string in case of a valid, non-empty string otherwise an empty string
65 */
66std::string decode_base64(const std::string &str)
67{
68 constexpr const char pad_char = '=';
69
70 // Handle empty string
71 if(str.empty())
72 {
73 return {};
74 }
75
76 // Base64 encoded string has size multiple of 4
77 if(str.length() % 4)
78 {
79 return {};
80 }
81
82 //
83 // Check encoded string padding
84 std::size_t padding = (str.rbegin()[0] == pad_char) + (str.rbegin()[1] == pad_char);
85 const int str_len = str.size();
86
87 // Reserve memory for the decoded string
88 // Note each 4 consecutive elements of 6-bit encode 3 bytes
89 std::string dec_b64;
90 dec_b64.reserve(((str_len / 4) * 3));
91
92 // Block decoding function (exclude padding)
93 int c = 0;
94 const int end = str_len - 4 - padding;
95 for(; c <= end; c += 4)
96 {
97 const int byte0 = b64_invtab[str[c]];
98 const int byte1 = b64_invtab[str[c + 1]];
99 const int byte2 = b64_invtab[str[c + 2]];
100 const int byte3 = b64_invtab[str[c + 3]];
101
102 dec_b64.push_back((byte0 << 2) | (byte1 >> 4));
103 dec_b64.push_back((byte1 << 4) | (byte2 >> 2));
104 dec_b64.push_back((byte2 << 6) | (byte3));
105 }
106
107 // Last step that might contain padding symbols
108 if(padding == 1)
109 {
110 const int byte0 = b64_invtab[str[c]];
111 const int byte1 = b64_invtab[str[c + 1]];
112 const int byte2 = b64_invtab[str[c + 2]];
113
114 dec_b64.push_back((byte0 << 2) | (byte1 >> 4));
115 dec_b64.push_back((byte1 << 4) | (byte2 >> 2));
116 }
117 else if(padding == 2)
118 {
119 const int byte0 = b64_invtab[str[c]];
120 const int byte1 = b64_invtab[str[c + 1]];
121
122 dec_b64.push_back((byte0 << 2) | (byte1 >> 4));
123 }
124
125 return dec_b64;
126}
127
128/** Decompress a zlib compressed string
129 *
130 * @param[in] str ZLib compressed string
131 *
132 * @return The decompressed string if successful, otherwise false.
133 */
134std::string decompress_zlib(const std::string &str)
135{
136 // Create and initialize decompression stream
137 z_stream ds{};
138 if(inflateInit(&ds) != Z_OK)
139 {
140 return std::string();
141 }
142 ds.avail_in = str.size();
143 ds.next_in = (Bytef *)str.data();
144
145 // Roll-over the string using a buffer and decompress
146 int status = Z_OK;
147 char roll_buff[16384];
148 std::string inflated_str;
149 do
150 {
151 ds.avail_out = sizeof(roll_buff);
152 ds.next_out = reinterpret_cast<Bytef *>(roll_buff);
153
154 status = inflate(&ds, 0);
155 if(inflated_str.size() < ds.total_out)
156 {
157 inflated_str.append(roll_buff, ds.total_out - inflated_str.size());
158 }
159 }
160 while(status == Z_OK);
161
162 // Finalize decompression stream
163 inflateEnd(&ds);
164 if(status != Z_STREAM_END)
165 {
166 return std::string();
167 }
168
169 return inflated_str;
170}
171} // namespace
172#endif /* ARM_COMPUTE_COMPRESSED_KERNELS */
173
174namespace arm_compute
175{
176namespace opencl
177{
178const std::map<std::string, std::string> ClKernelLibrary::_kernel_program_map =
179{
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100180 // Common Kernels
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100181 { "activation_layer", "common/activation_layer.cl" },
182 { "activation_layer_quant", "common/activation_layer_quant.cl" },
183 { "activation_layer_quant_f32", "common/activation_layer_quant.cl" },
184 { "arg_min_max_x", "common/arg_min_max.cl" },
185 { "arg_min_max_y", "common/arg_min_max.cl" },
186 { "arg_min_max_z", "common/arg_min_max.cl" },
187 { "arg_min_max_w", "common/arg_min_max.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100188 { "bitwise_or", "common/bitwise_op.cl" },
189 { "bitwise_and", "common/bitwise_op.cl" },
190 { "bitwise_xor", "common/bitwise_op.cl" },
191 { "bitwise_not", "common/bitwise_op.cl" },
192 { "bounding_box_transform", "common/bounding_box_transform.cl" },
193 { "bounding_box_transform_quantized", "common/bounding_box_transform_quantized.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100194 { "compare_equal", "common/comparisons.cl" },
195 { "compare_equal_quantized", "common/comparisons.cl" },
196 { "compare_notequal", "common/comparisons.cl" },
197 { "compare_notequal_quantized", "common/comparisons.cl" },
198 { "compare_greater", "common/comparisons.cl" },
199 { "compare_greater_quantized", "common/comparisons.cl" },
200 { "compare_greaterequal", "common/comparisons.cl" },
201 { "compare_greaterequal_quantized", "common/comparisons.cl" },
202 { "compare_less", "common/comparisons.cl" },
203 { "compare_less_quantized", "common/comparisons.cl" },
204 { "compare_lessequal", "common/comparisons.cl" },
205 { "compare_lessequal_quantized", "common/comparisons.cl" },
206 { "concatenate", "common/concatenate.cl" },
207 { "concatenate_width", "common/concatenate.cl" },
208 { "concatenate_height", "common/concatenate.cl" },
209 { "concatenate_width_x2", "common/concatenate.cl" },
210 { "concatenate_width_x4", "common/concatenate.cl" },
211 { "col2im", "common/col2im.cl" },
212 { "cast_down", "common/cast.cl" },
213 { "cast_up", "common/cast.cl" },
214 { "convert_fc_weights", "common/convert_fc_weights.cl" },
215 { "copy_tensor", "common/copy_tensor.cl" },
216 { "crop_tensor", "common/crop_tensor.cl" },
217 { "deconvolution_reshape", "common/deconvolution_layer.cl" },
218 { "deconvolution_upsample", "common/deconvolution_layer.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100219 { "dequantization_layer", "common/dequantization_layer.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100220 { "elementwise_operation_ADD", "common/elementwise_operation.cl" },
221 { "elementwise_operation_SUB", "common/elementwise_operation.cl" },
222 { "elementwise_operation_MAX", "common/elementwise_operation.cl" },
223 { "elementwise_operation_MIN", "common/elementwise_operation.cl" },
224 { "elementwise_operation_DIV", "common/elementwise_operation.cl" },
225 { "elementwise_operation_SQUARED_DIFF", "common/elementwise_operation.cl" },
226 { "elementwise_operation_POWER", "common/elementwise_operation.cl" },
227 { "elementwise_operation_PRELU", "common/elementwise_operation.cl" },
228 { "elementwise_operation_AND", "common/elementwise_operation.cl" },
229 { "elementwise_operation_OR", "common/elementwise_operation.cl" },
230 { "elementwise_operation_ADD_quantized", "common/elementwise_operation_quantized.cl" },
231 { "elementwise_operation_SUB_quantized", "common/elementwise_operation_quantized.cl" },
232 { "elementwise_operation_MAX_quantized", "common/elementwise_operation_quantized.cl" },
233 { "elementwise_operation_MIN_quantized", "common/elementwise_operation_quantized.cl" },
234 { "elementwise_operation_DIV_quantized", "common/elementwise_operation_quantized.cl" },
235 { "elementwise_operation_SQUARED_DIFF_quantized", "common/elementwise_operation_quantized.cl" },
236 { "elementwise_operation_PRELU_quantized", "common/elementwise_operation_quantized.cl" },
237 { "elementwise_unary", "common/elementwise_unary.cl" },
238 { "fft_digit_reverse_axis_0", "common/fft_digit_reverse.cl" },
239 { "fft_digit_reverse_axis_1", "common/fft_digit_reverse.cl" },
240 { "fft_radix_2_first_stage_axis_0", "common/fft.cl" },
241 { "fft_radix_2_first_stage_axis_1", "common/fft.cl" },
242 { "fft_radix_2_axis_0", "common/fft.cl" },
243 { "fft_radix_2_axis_1", "common/fft.cl" },
244 { "fft_radix_3_first_stage_axis_0", "common/fft.cl" },
245 { "fft_radix_3_first_stage_axis_1", "common/fft.cl" },
246 { "fft_radix_3_axis_0", "common/fft.cl" },
247 { "fft_radix_3_axis_1", "common/fft.cl" },
248 { "fft_radix_4_first_stage_axis_0", "common/fft.cl" },
249 { "fft_radix_4_first_stage_axis_1", "common/fft.cl" },
250 { "fft_radix_4_axis_0", "common/fft.cl" },
251 { "fft_radix_4_axis_1", "common/fft.cl" },
252 { "fft_radix_5_first_stage_axis_0", "common/fft.cl" },
253 { "fft_radix_5_first_stage_axis_1", "common/fft.cl" },
254 { "fft_radix_5_axis_0", "common/fft.cl" },
255 { "fft_radix_5_axis_1", "common/fft.cl" },
256 { "fft_radix_7_first_stage_axis_0", "common/fft.cl" },
257 { "fft_radix_7_first_stage_axis_1", "common/fft.cl" },
258 { "fft_radix_7_axis_0", "common/fft.cl" },
259 { "fft_radix_7_axis_1", "common/fft.cl" },
260 { "fft_radix_8_first_stage_axis_0", "common/fft.cl" },
261 { "fft_radix_8_first_stage_axis_1", "common/fft.cl" },
262 { "fft_radix_8_axis_0", "common/fft.cl" },
263 { "fft_radix_8_axis_1", "common/fft.cl" },
264 { "fft_scale_conj", "common/fft_scale.cl" },
265 { "fill_image_borders_constant", "common/fill_border.cl" },
266 { "fill_image_borders_replicate", "common/fill_border.cl" },
267 { "floor_layer", "common/floor.cl" },
268 { "fuse_batchnormalization_layer", "common/batchnormalization_layer.cl" },
269 { "gather", "common/gather.cl" },
270 { "gemm_ma_f16", "common/gemm.cl" },
271 { "gemm_ma_f32", "common/gemm.cl" },
272 { "gemm_mv", "common/gemv.cl" },
273 { "gemm_mv_quantized", "common/gemv.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100274 { "gemm_mm_native", "common/gemm.cl" },
SiCongLiafa19722021-10-24 19:12:33 +0100275 { "gemm_mm_native_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_native.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100276 { "gemm_mm_reshaped_lhs_nt_rhs_t", "common/gemm.cl" },
277 { "gemm_mm_reshaped_lhs_nt_rhs_t_texture", "common/gemm.cl" },
278 { "gemm_mm_reshaped_lhs_t_rhs_nt", "common/gemm.cl" },
279 { "gemm_mm_reshaped_lhs_t_rhs_nt_texture", "common/gemm.cl" },
SiCongLi1af54162021-10-06 15:25:57 +0100280 { "gemm_mm_reshaped_lhs_nt_rhs_t_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.cl" },
281 { "gemm_mm_reshaped_lhs_nt_rhs_t_texture_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.cl" },
282 { "gemm_mm_reshaped_lhs_t_rhs_nt_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.cl" },
283 { "gemm_mm_reshaped_lhs_t_rhs_nt_texture_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.cl" },
Ramy Elgammal451c3092022-02-01 23:01:27 +0000284 { "gemm_mm_reshaped_only_rhs_nt", "common/gemm.cl" },
285 { "gemm_mm_reshaped_only_rhs_nt_texture", "common/gemm.cl" },
286 { "gemm_mm_reshaped_only_rhs_t", "common/gemm.cl" },
287 { "gemm_mm_reshaped_only_rhs_t_texture", "common/gemm.cl" },
288 { "gemm_mm_reshaped_only_rhs_nt_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl" },
289 { "gemm_mm_reshaped_only_rhs_nt_texture_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl" },
290 { "gemm_mm_reshaped_only_rhs_t_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl" },
291 { "gemm_mm_reshaped_only_rhs_t_texture_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100292 { "gemm_lc_vm_f32", "common/gemm.cl" },
ramelg019cca5922021-11-11 10:05:00 +0000293 { "gemm_reshape_lhs_matrix_nt", "common/gemm_utils.cl" },
294 { "gemm_reshape_lhs_matrix_t", "common/gemm_utils.cl" },
295 { "gemm_reshape_rhs_matrix_nt", "common/gemm_utils.cl" },
296 { "gemm_reshape_rhs_matrix_t", "common/gemm_utils.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100297 { "gemmlowp_matrix_a_reduction", "common/gemmlowp.cl" },
298 { "gemmlowp_matrix_a_reduction_dot8", "common/gemmlowp.cl" },
299 { "gemmlowp_matrix_b_reduction", "common/gemmlowp.cl" },
300 { "gemmlowp_mm_native", "common/gemmlowp.cl" },
301 { "gemmlowp_mm_reshaped_lhs_nt_rhs_t", "common/gemmlowp.cl" },
302 { "gemmlowp_mm_reshaped_only_rhs_t", "common/gemmlowp.cl" },
303 { "gemmlowp_mm_reshaped_only_rhs_t_fused_output_stage_fixedpoint", "common/gemmlowp.cl" },
304 { "gemmlowp_offset_contribution", "common/gemmlowp.cl" },
305 { "gemmlowp_offset_contribution_quantize_down", "common/gemmlowp.cl" },
306 { "gemmlowp_offset_contribution_quantize_down_fixedpoint", "common/gemmlowp.cl" },
307 { "gemmlowp_output_stage_quantize_down", "common/gemmlowp.cl" },
308 { "gemmlowp_output_stage_quantize_down_fixedpoint", "common/gemmlowp.cl" },
309 { "gemmlowp_output_stage_quantize_down_fixedpoint_qsymm16", "common/gemmlowp.cl" },
310 { "gemmlowp_output_stage_quantize_down_float", "common/gemmlowp.cl" },
311 { "generate_proposals_compute_all_anchors", "common/generate_proposals.cl" },
312 { "generate_proposals_compute_all_anchors_quantized", "common/generate_proposals_quantized.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100313 { "instance_normalization", "common/instance_normalization.cl" },
314 { "compute_mean_var", "common/instance_normalization.cl" },
315 { "l2_normalize_x", "common/l2_normalize.cl" },
316 { "l2_normalize_y", "common/l2_normalize.cl" },
317 { "l2_normalize_z", "common/l2_normalize.cl" },
318 { "max_unpooling_layer_2", "common/unpooling_layer.cl" },
319 { "mean_stddev_normalization", "common/mean_stddev_normalization.cl" },
320 { "memset", "common/memset.cl" },
321 { "minmax_layer", "common/minmax_layer.cl" },
322 { "non_max_suppression", "common/nonmax.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100323 { "pad_layer_constant", "common/pad_layer.cl" },
324 { "pad_layer_symmetric_reflect", "common/pad_layer.cl" },
325 { "permute", "common/permute.cl" },
326 { "pixelwise_mul_complex", "common/pixelwise_mul_float.cl" },
327 { "pixelwise_mul_float", "common/pixelwise_mul_float.cl" },
328 { "pixelwise_mul_int", "common/pixelwise_mul_int.cl" },
329 { "pixelwise_mul_quantized", "common/pixelwise_mul_int.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100330 { "qlstm_layer_normalization", "common/qlstm_layer_normalization.cl" },
331 { "quantization_layer", "common/quantization_layer.cl" },
332 { "range", "common/range.cl" },
333 { "range_quantized", "common/range.cl" },
334 { "reduction_operation_x", "common/reduction_operation.cl" },
335 { "reduction_operation_non_parallel_x", "common/reduction_operation.cl" },
336 { "reduction_operation_y", "common/reduction_operation.cl" },
337 { "reduction_operation_z", "common/reduction_operation.cl" },
338 { "reduction_operation_w", "common/reduction_operation.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100339 { "reshape_layer", "common/reshape_layer.cl" },
340 { "reshape_to_columns", "common/convolution_layer.cl" },
341 { "reverse", "common/reverse.cl" },
342 { "roi_align_layer", "common/roi_align_layer.cl" },
343 { "roi_align_layer_quantized", "common/roi_align_layer_quantized.cl" },
344 { "roi_pooling_layer", "common/roi_pooling_layer.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100345 { "select_same_rank", "common/select.cl" },
346 { "select_different_rank_2", "common/select.cl" },
347 { "select_different_rank_n", "common/select.cl" },
348 { "softmax_layer_norm", "common/softmax_layer.cl" },
349 { "softmax_layer_norm_quantized", "common/softmax_layer_quantized.cl" },
350 { "softmax_layer_max_shift_exp_sum_quantized_serial", "common/softmax_layer_quantized.cl" },
351 { "softmax_layer_max_shift_exp_sum_quantized_parallel", "common/softmax_layer_quantized.cl" },
352 { "softmax_layer_max_shift_exp_sum_serial", "common/softmax_layer.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100353 { "softmax_layer_max_shift_exp_sum_parallel", "common/softmax_layer.cl" },
354 { "stack_layer", "common/stack_layer.cl" },
355 { "strided_slice", "common/slice_ops.cl" },
356 { "tile", "common/tile.cl" },
357 { "transpose", "common/transpose.cl" },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100358#ifdef ENABLE_NCHW_KERNELS
359 { "batch_to_space_nchw", "nchw/batch_to_space.cl" },
360 { "batch_to_space_static_nchw", "nchw/batch_to_space.cl" },
361 { "batchnormalization_layer_nchw", "nchw/batchnormalization_layer.cl" },
362 { "channel_shuffle_nchw", "nchw/channel_shuffle.cl" },
363 { "depth_to_space_nchw", "nchw/depth_to_space.cl" },
364 { "dequantization_layer_per_channel_nchw", "nchw/dequantization_layer.cl" },
365 { "direct_convolution1x1", "nchw/direct_convolution1x1.cl" },
Adnan AlSinan30124352021-12-02 19:12:20 +0000366 { "direct_convolution_nchw", "nchw/direct_convolution.cl" },
367
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100368 { "im2col1x1_stridex1_nchw", "nchw/im2col.cl" },
369 { "im2col3x3_nchw", "nchw/im2col.cl" },
370 { "im2col5x5_nchw", "nchw/im2col.cl" },
371 { "im2col11x11_padx0_pady0_nchw", "nchw/im2col.cl" },
372 { "im2col_generic_nchw", "nchw/im2col.cl" },
373 { "im2col_generic_padx0_pady0_nchw", "nchw/im2col.cl" },
374 { "normalization_layer_cross_map_nchw", "nchw/normalization_layer.cl" },
375 { "normalization_layer_in_map_nchw", "nchw/normalization_layer.cl" },
376 { "normalize_planar_yuv_layer_nchw", "nchw/normalize_planar_yuv_layer.cl" },
377 { "normalize_planar_yuv_layer_q8_nchw", "nchw/normalize_planar_yuv_layer_quantized.cl" },
378 { "pooling_layer_MxN_nchw", "nchw/pooling_layer.cl" },
Giorgio Arena8fce4962021-09-01 14:05:00 +0100379 { "pooling_layer_2_nchw_indices", "nchw/pooling_layer.cl" },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100380 { "prior_box_layer_nchw", "nchw/prior_box_layer.cl" },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100381 { "reorg_layer_nchw", "nchw/reorg_layer.cl" },
382 { "scale_nearest_neighbour_nchw", "nchw/scale.cl" },
383 { "scale_bilinear_nchw", "nchw/scale.cl" },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100384 { "space_to_batch_nchw", "nchw/space_to_batch.cl" },
385 { "space_to_batch_static_nchw", "nchw/space_to_batch.cl" },
386 { "space_to_depth_nchw", "nchw/space_to_depth.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100387 { "upsample_layer_nchw", "nchw/upsample_layer.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100388 { "winograd_filter_transform_2x2_3x3_nchw", "nchw/winograd_filter_transform.cl" },
389 { "winograd_filter_transform_2x1_3x1_nchw", "nchw/winograd_filter_transform.cl" },
390 { "winograd_filter_transform_1x2_1x3_nchw", "nchw/winograd_filter_transform.cl" },
391 { "winograd_filter_transform_4x4_3x3_nchw", "nchw/winograd_filter_transform.cl" },
392 { "winograd_filter_transform_4x1_3x1_nchw", "nchw/winograd_filter_transform.cl" },
393 { "winograd_filter_transform_1x4_1x3_nchw", "nchw/winograd_filter_transform.cl" },
394 { "winograd_filter_transform_4x4_5x5_nchw", "nchw/winograd_filter_transform.cl" },
395 { "winograd_filter_transform_4x1_5x1_nchw", "nchw/winograd_filter_transform.cl" },
396 { "winograd_filter_transform_1x4_1x5_nchw", "nchw/winograd_filter_transform.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100397 { "winograd_input_transform_2x2_3x3_stepz1_nchw", "nchw/winograd_input_transform.cl" },
398 { "winograd_input_transform_2x2_3x3_stepz2_nchw", "nchw/winograd_input_transform.cl" },
399 { "winograd_input_transform_2x1_3x1_stepz1_nchw", "nchw/winograd_input_transform.cl" },
400 { "winograd_input_transform_2x1_3x1_stepz2_nchw", "nchw/winograd_input_transform.cl" },
401 { "winograd_input_transform_1x2_1x3_stepz1_nchw", "nchw/winograd_input_transform.cl" },
402 { "winograd_input_transform_1x2_1x3_stepz2_nchw", "nchw/winograd_input_transform.cl" },
403 { "winograd_input_transform_4x4_3x3_stepz1_nchw", "nchw/winograd_input_transform.cl" },
404 { "winograd_input_transform_4x1_3x1_stepz1_nchw", "nchw/winograd_input_transform.cl" },
405 { "winograd_input_transform_1x4_1x3_stepz1_nchw", "nchw/winograd_input_transform.cl" },
406 { "winograd_input_transform_4x4_5x5_stepz1_nchw", "nchw/winograd_input_transform.cl" },
407 { "winograd_input_transform_4x1_5x1_stepz1_nchw", "nchw/winograd_input_transform.cl" },
408 { "winograd_input_transform_1x4_1x5_stepz1_nchw", "nchw/winograd_input_transform.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100409 { "winograd_output_transform_2x2_3x3_nchw", "nchw/winograd_output_transform.cl" },
410 { "winograd_output_transform_2x1_3x1_nchw", "nchw/winograd_output_transform.cl" },
411 { "winograd_output_transform_1x2_1x3_nchw", "nchw/winograd_output_transform.cl" },
412 { "winograd_output_transform_4x4_3x3_nchw", "nchw/winograd_output_transform.cl" },
413 { "winograd_output_transform_4x1_3x1_nchw", "nchw/winograd_output_transform.cl" },
414 { "winograd_output_transform_1x4_1x3_nchw", "nchw/winograd_output_transform.cl" },
415 { "winograd_output_transform_4x4_5x5_nchw", "nchw/winograd_output_transform.cl" },
416 { "winograd_output_transform_4x1_5x1_nchw", "nchw/winograd_output_transform.cl" },
417 { "winograd_output_transform_1x4_1x5_nchw", "nchw/winograd_output_transform.cl" },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100418#endif /* ENABLE_NCHW_KERNELS */
419#ifdef ENABLE_NHWC_KERNELS
420 { "batch_to_space_nhwc", "nhwc/batch_to_space.cl" },
421 { "batch_to_space_static_nhwc", "nhwc/batch_to_space.cl" },
422 { "batchnormalization_layer_nhwc", "nhwc/batchnormalization_layer.cl" },
423 { "channel_shuffle_nhwc", "nhwc/channel_shuffle.cl" },
424 { "depth_to_space_nhwc", "nhwc/depth_to_space.cl" },
425 { "dequantization_layer_per_channel_nhwc", "nhwc/dequantization_layer.cl" },
426 { "dwc_native_fp_nhwc", "nhwc/dwc_native_fp_nhwc.cl" },
427 { "dwc_native_quantized_nhwc", "nhwc/dwc_native_quantized_nhwc.cl" },
428 { "direct_convolution_nhwc", "nhwc/direct_convolution.cl" },
Giorgio Arena945ae9e2021-10-13 11:13:04 +0100429 { "direct_convolution3d_ndhwc", "nhwc/direct_convolution3d.cl" },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100430 { "im2col3x3_nhwc", "nhwc/im2col.cl" },
431 { "im2col9x9_nhwc", "nhwc/im2col.cl" },
432 { "im2col_generic_nhwc", "nhwc/im2col.cl" },
433 { "normalization_layer_cross_map_nhwc", "nhwc/normalization_layer.cl" },
434 { "normalization_layer_in_map_nhwc", "nhwc/normalization_layer.cl" },
435 { "normalize_planar_yuv_layer_nhwc", "nhwc/normalize_planar_yuv_layer.cl" },
436 { "normalize_planar_yuv_layer_q8_nhwc", "nhwc/normalize_planar_yuv_layer_quantized.cl" },
437 { "pooling_layer_MxN_nhwc", "nhwc/pooling_layer.cl" },
438 { "pooling_layer_2x2_nhwc", "nhwc/pooling_layer.cl" },
439 { "pooling_layer_MxN_quantized_nhwc", "nhwc/pooling_layer_quantized.cl" },
ramelg0137515692022-02-26 22:06:20 +0000440 { "pooling_3d_layer_MxN_ndhwc", "nhwc/pooling_3d_layer.cl" },
Mohammed Suhail Munshi5e549fa2022-03-16 11:14:06 +0000441 { "pooling_3d_layer_MxN_ndhwc_quantized", "nhwc/pooling_3d_layer_quantized.cl" },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100442 { "reorg_layer_nhwc", "nhwc/reorg_layer.cl" },
443 { "scale_nearest_neighbour_nhwc", "nhwc/scale.cl" },
444 { "scale_bilinear_nhwc", "nhwc/scale.cl" },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100445 { "space_to_batch_nhwc", "nhwc/space_to_batch.cl" },
446 { "space_to_batch_static_nhwc", "nhwc/space_to_batch.cl" },
447 { "space_to_depth_nhwc", "nhwc/space_to_depth.cl" },
448 { "upsample_layer_nhwc", "nhwc/upsample_layer.cl" },
449 { "winograd_filter_transform_4x1_3x1_nhwc", "nhwc/winograd_filter_transform.cl" },
450 { "winograd_filter_transform_1x4_1x3_nhwc", "nhwc/winograd_filter_transform.cl" },
451 { "winograd_filter_transform_4x4_3x3_nhwc", "nhwc/winograd_filter_transform.cl" },
452 { "winograd_filter_transform_4x4_5x5_nhwc", "nhwc/winograd_filter_transform.cl" },
453 { "winograd_filter_transform_4x1_5x1_nhwc", "nhwc/winograd_filter_transform.cl" },
454 { "winograd_filter_transform_1x4_1x5_nhwc", "nhwc/winograd_filter_transform.cl" },
455 { "winograd_filter_transform_2x2_7x7_nhwc", "nhwc/winograd_filter_transform.cl" },
456 { "winograd_filter_transform_2x1_7x1_nhwc", "nhwc/winograd_filter_transform.cl" },
457 { "winograd_filter_transform_1x2_1x7_nhwc", "nhwc/winograd_filter_transform.cl" },
458 { "winograd_input_transform_4x1_3x1_stepz1_nhwc", "nhwc/winograd_input_transform.cl" },
459 { "winograd_input_transform_1x4_1x3_stepz1_nhwc", "nhwc/winograd_input_transform.cl" },
460 { "winograd_input_transform_4x4_3x3_stepz1_nhwc", "nhwc/winograd_input_transform.cl" },
461 { "winograd_input_transform_4x4_5x5_stepz1_nhwc", "nhwc/winograd_input_transform.cl" },
462 { "winograd_input_transform_4x1_5x1_stepz1_nhwc", "nhwc/winograd_input_transform.cl" },
463 { "winograd_input_transform_1x4_1x5_stepz1_nhwc", "nhwc/winograd_input_transform.cl" },
464 { "winograd_input_transform_2x2_7x7_stepz1_nhwc", "nhwc/winograd_input_transform.cl" },
465 { "winograd_input_transform_2x1_7x1_stepz1_nhwc", "nhwc/winograd_input_transform.cl" },
466 { "winograd_input_transform_1x2_1x7_stepz1_nhwc", "nhwc/winograd_input_transform.cl" },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100467 { "winograd_output_transform_4x1_3x1_nhwc", "nhwc/winograd_output_transform.cl" },
468 { "winograd_output_transform_1x4_1x3_nhwc", "nhwc/winograd_output_transform.cl" },
469 { "winograd_output_transform_4x4_3x3_nhwc", "nhwc/winograd_output_transform.cl" },
470 { "winograd_output_transform_4x4_5x5_nhwc", "nhwc/winograd_output_transform.cl" },
471 { "winograd_output_transform_4x1_5x1_nhwc", "nhwc/winograd_output_transform.cl" },
472 { "winograd_output_transform_1x4_1x5_nhwc", "nhwc/winograd_output_transform.cl" },
473 { "winograd_output_transform_2x2_7x7_nhwc", "nhwc/winograd_output_transform.cl" },
474 { "winograd_output_transform_2x1_7x1_nhwc", "nhwc/winograd_output_transform.cl" },
475 { "winograd_output_transform_1x2_1x7_nhwc", "nhwc/winograd_output_transform.cl" },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100476#endif /* ENABLE_NHWC_KERNELS */
Georgios Pinitas908f6162021-05-04 10:11:09 +0100477};
478
479const std::map<std::string, std::string> ClKernelLibrary::_program_source_map =
480{
481#ifdef EMBEDDED_KERNELS
482 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100483 "common/activation_layer.cl",
484#include "./cl_kernels/common/activation_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100485 },
486 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100487 "common/activation_layer_quant.cl",
488#include "./cl_kernels/common/activation_layer_quant.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100489 },
490 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100491 "common/arg_min_max.cl",
492#include "./cl_kernels/common/arg_min_max.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100493 },
494 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100495 "common/bitwise_op.cl",
496#include "./cl_kernels/common/bitwise_op.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100497 },
498 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100499 "common/bounding_box_transform.cl",
500#include "./cl_kernels/common/bounding_box_transform.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100501 },
502 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100503 "common/bounding_box_transform_quantized.cl",
504#include "./cl_kernels/common/bounding_box_transform_quantized.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100505 },
506 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100507 "common/col2im.cl",
508#include "./cl_kernels/common/col2im.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100509 },
510 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100511 "common/comparisons.cl",
512#include "./cl_kernels/common/comparisons.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100513 },
514 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100515 "common/concatenate.cl",
516#include "./cl_kernels/common/concatenate.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100517 },
518 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100519 "common/convert_fc_weights.cl",
520#include "./cl_kernels/common/convert_fc_weights.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100521 },
522 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100523 "common/convolution_layer.cl",
524#include "./cl_kernels/common/convolution_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100525 },
526 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100527 "common/copy_tensor.cl",
528#include "./cl_kernels/common/copy_tensor.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100529 },
530 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100531 "common/crop_tensor.cl",
532#include "./cl_kernels/common/crop_tensor.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100533 },
534 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100535 "common/deconvolution_layer.cl",
536#include "./cl_kernels/common/deconvolution_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100537 },
538 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100539 "common/cast.cl",
540#include "./cl_kernels/common/cast.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100541 },
542 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100543 "common/dequantization_layer.cl",
544#include "./cl_kernels/common/dequantization_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100545 },
546 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100547 "common/elementwise_operation.cl",
548#include "./cl_kernels/common/elementwise_operation.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100549 },
550 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100551 "common/elementwise_operation_quantized.cl",
552#include "./cl_kernels/common/elementwise_operation_quantized.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100553 },
554 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100555 "common/elementwise_unary.cl",
556#include "./cl_kernels/common/elementwise_unary.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100557 },
558 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100559 "common/fft.cl",
560#include "./cl_kernels/common/fft.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100561 },
562 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100563 "common/fft_digit_reverse.cl",
564#include "./cl_kernels/common/fft_digit_reverse.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100565 },
566 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100567 "common/fft_scale.cl",
568#include "./cl_kernels/common/fft_scale.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100569 },
570 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100571 "common/fill_border.cl",
572#include "./cl_kernels/common/fill_border.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100573 },
574 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100575 "common/floor.cl",
576#include "./cl_kernels/common/floor.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100577 },
578 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100579 "common/gather.cl",
580#include "./cl_kernels/common/gather.clembed"
581 },
582 {
583 "common/gemm.cl",
584#include "./cl_kernels/common/gemm.clembed"
585 },
586 {
ramelg019cca5922021-11-11 10:05:00 +0000587 "common/gemm_utils.cl",
588#include "./cl_kernels/common/gemm_utils.clembed"
589 },
590 {
Giorgio Arena892b70a2022-03-30 12:23:10 +0100591 "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/fp_post_ops_act_eltwise_op_act.h",
592#include "./cl_kernels/common/experimental/gemm_fused_post_ops/act_eltwise_op_act/fp_post_ops_act_eltwise_op_act.hembed"
593 },
594 {
SiCong Lica364df2022-04-13 15:48:19 +0100595 "common/experimental/gemm_fused_post_ops/fp_mixed_precision_helpers.h",
596#include "./cl_kernels/common/experimental/gemm_fused_post_ops/fp_mixed_precision_helpers.hembed"
597 },
598 {
SiCongLiafa19722021-10-24 19:12:33 +0100599 "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_native.cl",
600#include "./cl_kernels/common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_native.clembed"
601 },
602 {
SiCongLi1af54162021-10-06 15:25:57 +0100603 "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.cl",
604#include "./cl_kernels/common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.clembed"
605 },
606 {
Ramy Elgammal451c3092022-02-01 23:01:27 +0000607 "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl",
608#include "./cl_kernels/common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.clembed"
609 },
610 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100611 "common/gemmlowp.cl",
612#include "./cl_kernels/common/gemmlowp.clembed"
613 },
614 {
615 "common/gemv.cl",
616#include "./cl_kernels/common/gemv.clembed"
617 },
618 {
619 "common/generate_proposals.cl",
620#include "./cl_kernels/common/generate_proposals.clembed"
621 },
622 {
623 "common/generate_proposals_quantized.cl",
624#include "./cl_kernels/common/generate_proposals_quantized.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100625 },
626 {
Giorgio Arena892b70a2022-03-30 12:23:10 +0100627 "gemm_helpers.h",
628#include "./cl_kernels/gemm_helpers.hembed"
629 },
630 {
Georgios Pinitas908f6162021-05-04 10:11:09 +0100631 "helpers.h",
632#include "./cl_kernels/helpers.hembed"
633 },
634 {
635 "helpers_asymm.h",
636#include "./cl_kernels/helpers_asymm.hembed"
637 },
638 {
Giorgio Arena892b70a2022-03-30 12:23:10 +0100639 "repeat.h",
640#include "./cl_kernels/repeat.hembed"
641 },
642 {
SiCong Lica364df2022-04-13 15:48:19 +0100643 "tile_helpers.h",
644#include "./cl_kernels/tile_helpers.hembed"
645 },
646 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100647 "common/instance_normalization.cl",
648#include "./cl_kernels/common/instance_normalization.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100649 },
650 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100651 "common/l2_normalize.cl",
652#include "./cl_kernels/common/l2_normalize.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100653 },
654 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100655 "common/mean_stddev_normalization.cl",
656#include "./cl_kernels/common/mean_stddev_normalization.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100657 },
658 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100659 "common/memset.cl",
660#include "./cl_kernels/common/memset.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100661 },
662 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100663 "common/minmax_layer.cl",
664#include "./cl_kernels/common/minmax_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100665 },
666 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100667 "common/nonmax.cl",
668#include "./cl_kernels/common/nonmax.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100669 },
670 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100671 "common/batchnormalization_layer.cl",
672#include "./cl_kernels/common/batchnormalization_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100673 },
674 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100675 "common/pad_layer.cl",
676#include "./cl_kernels/common/pad_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100677 },
678 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100679 "common/permute.cl",
680#include "./cl_kernels/common/permute.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100681 },
682 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100683 "common/pixelwise_mul_float.cl",
684#include "./cl_kernels/common/pixelwise_mul_float.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100685 },
686 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100687 "common/pixelwise_mul_int.cl",
688#include "./cl_kernels/common/pixelwise_mul_int.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100689 },
690 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100691 "common/qlstm_layer_normalization.cl",
692#include "./cl_kernels/common/qlstm_layer_normalization.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100693 },
694 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100695 "common/quantization_layer.cl",
696#include "./cl_kernels/common/quantization_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100697 },
698 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100699 "common/range.cl",
700#include "./cl_kernels/common/range.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100701 },
702 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100703 "common/reduction_operation.cl",
704#include "./cl_kernels/common/reduction_operation.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100705 },
706 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100707 "common/reshape_layer.cl",
708#include "./cl_kernels/common/reshape_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100709 },
710 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100711 "common/reverse.cl",
712#include "./cl_kernels/common/reverse.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100713 },
714 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100715 "common/roi_align_layer.cl",
716#include "./cl_kernels/common/roi_align_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100717 },
718 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100719 "common/roi_align_layer_quantized.cl",
720#include "./cl_kernels/common/roi_align_layer_quantized.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100721 },
722 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100723 "common/roi_pooling_layer.cl",
724#include "./cl_kernels/common/roi_pooling_layer.clembed"
725 },
726 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100727 "common/select.cl",
728#include "./cl_kernels/common/select.clembed"
729 },
730 {
731 "common/softmax_layer.cl",
732#include "./cl_kernels/common/softmax_layer.clembed"
733 },
734 {
735 "common/softmax_layer_quantized.cl",
736#include "./cl_kernels/common/softmax_layer_quantized.clembed"
737 },
738 {
739 "common/slice_ops.cl",
740#include "./cl_kernels/common/slice_ops.clembed"
741 },
742 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100743 "common/stack_layer.cl",
744#include "./cl_kernels/common/stack_layer.clembed"
745 },
746 {
747 "common/tile.cl",
748#include "./cl_kernels/common/tile.clembed"
749 },
750 {
751 "common/transpose.cl",
752#include "./cl_kernels/common/transpose.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100753 },
754 {
755 "types.h",
756#include "./cl_kernels/types.hembed"
757 },
758 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100759 "common/unpooling_layer.cl",
760#include "./cl_kernels/common/unpooling_layer.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100761 },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100762#ifdef ENABLE_NCHW_KERNELS
763 {
764 "nchw/batch_to_space.cl",
765#include "./cl_kernels/nchw/batch_to_space.clembed"
766 },
767 {
768 "nchw/channel_shuffle.cl",
769#include "./cl_kernels/nchw/channel_shuffle.clembed"
770 },
771 {
772 "nchw/upsample_layer.cl",
773#include "./cl_kernels/nchw/upsample_layer.clembed"
774 },
775 {
776 "nchw/depth_to_space.cl",
777#include "./cl_kernels/nchw/depth_to_space.clembed"
778 },
779 {
780 "nchw/dequantization_layer.cl",
781#include "./cl_kernels/nchw/dequantization_layer.clembed"
782 },
783 {
Adnan AlSinan30124352021-12-02 19:12:20 +0000784 "nchw/direct_convolution.cl",
785#include "./cl_kernels/nchw/direct_convolution.clembed"
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100786 },
787 {
788 "nchw/im2col.cl",
789#include "./cl_kernels/nchw/im2col.clembed"
790 },
791 {
792 "nchw/normalization_layer.cl",
793#include "./cl_kernels/nchw/normalization_layer.clembed"
794 },
795 {
796 "nchw/normalize_planar_yuv_layer.cl",
797#include "./cl_kernels/nchw/normalize_planar_yuv_layer.clembed"
798 },
799 {
800 "nchw/normalize_planar_yuv_layer_quantized.cl",
801#include "./cl_kernels/nchw/normalize_planar_yuv_layer_quantized.clembed"
802 },
803 {
804 "nchw/batchnormalization_layer.cl",
805#include "./cl_kernels/nchw/batchnormalization_layer.clembed"
806 },
807 {
808 "nchw/pooling_layer.cl",
809#include "./cl_kernels/nchw/pooling_layer.clembed"
810 },
811 {
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100812 "nchw/prior_box_layer.cl",
813#include "./cl_kernels/nchw/prior_box_layer.clembed"
814 },
815 {
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100816 "nchw/reorg_layer.cl",
817#include "./cl_kernels/nchw/reorg_layer.clembed"
818 },
819 {
820 "nchw/scale.cl",
821#include "./cl_kernels/nchw/scale.clembed"
822 },
823 {
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100824 "nchw/space_to_batch.cl",
825#include "./cl_kernels/nchw/space_to_batch.clembed"
826 },
827 {
828 "nchw/space_to_depth.cl",
829#include "./cl_kernels/nchw/space_to_depth.clembed"
830 },
Georgios Pinitas908f6162021-05-04 10:11:09 +0100831 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100832 "nchw/winograd_filter_transform.cl",
833#include "./cl_kernels/nchw/winograd_filter_transform.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100834 },
835 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100836 "nchw/winograd_input_transform.cl",
837#include "./cl_kernels/nchw/winograd_input_transform.clembed"
838 },
839 {
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100840 "nchw/winograd_output_transform.cl",
841#include "./cl_kernels/nchw/winograd_output_transform.clembed"
842 },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100843#endif /* ENABLE_NCHW_KERNELS */
844
845#ifdef ENABLE_NHWC_KERNELS
846 {
847 "nhwc/batch_to_space.cl",
848#include "./cl_kernels/nhwc/batch_to_space.clembed"
849 },
850 {
851 "nhwc/channel_shuffle.cl",
852#include "./cl_kernels/nhwc/channel_shuffle.clembed"
853 },
854 {
855 "nhwc/upsample_layer.cl",
856#include "./cl_kernels/nhwc/upsample_layer.clembed"
857 },
858 {
859 "nhwc/depth_to_space.cl",
860#include "./cl_kernels/nhwc/depth_to_space.clembed"
861 },
862 {
863 "nhwc/dequantization_layer.cl",
864#include "./cl_kernels/nhwc/dequantization_layer.clembed"
865 },
866 {
867 "nhwc/direct_convolution.cl",
868#include "./cl_kernels/nhwc/direct_convolution.clembed"
869 },
870 {
Giorgio Arena945ae9e2021-10-13 11:13:04 +0100871 "nhwc/direct_convolution3d.cl",
872#include "./cl_kernels/nhwc/direct_convolution3d.clembed"
873 },
874 {
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100875 "nhwc/dwc_native_fp_nhwc.cl",
876#include "./cl_kernels/nhwc/dwc_native_fp_nhwc.clembed"
877 },
878 {
879 "nhwc/dwc_native_quantized_nhwc.cl",
880#include "./cl_kernels/nhwc/dwc_native_quantized_nhwc.clembed"
881 },
882 {
883 "nhwc/normalization_layer.cl",
884#include "./cl_kernels/nhwc/normalization_layer.clembed"
885 },
886 {
887 "nhwc/normalize_planar_yuv_layer.cl",
888#include "./cl_kernels/nhwc/normalize_planar_yuv_layer.clembed"
889 },
890 {
891 "nhwc/normalize_planar_yuv_layer_quantized.cl",
892#include "./cl_kernels/nhwc/normalize_planar_yuv_layer_quantized.clembed"
893 },
894 {
895 "nhwc/im2col.cl",
896#include "./cl_kernels/nhwc/im2col.clembed"
897 },
898 {
899 "nhwc/batchnormalization_layer.cl",
900#include "./cl_kernels/nhwc/batchnormalization_layer.clembed"
901 },
902 {
903 "nhwc/pooling_layer.cl",
904#include "./cl_kernels/nhwc/pooling_layer.clembed"
905 },
906 {
ramelg0137515692022-02-26 22:06:20 +0000907 "nhwc/pooling_3d_layer.cl",
908#include "./cl_kernels/nhwc/pooling_3d_layer.clembed"
909 },
910 {
Mohammed Suhail Munshi5e549fa2022-03-16 11:14:06 +0000911 "nhwc/pooling_3d_layer_quantized.cl",
912#include "./cl_kernels/nhwc/pooling_3d_layer_quantized.clembed"
913 },
914 {
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100915 "nhwc/pooling_layer_quantized.cl",
916#include "./cl_kernels/nhwc/pooling_layer_quantized.clembed"
917 },
918 {
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100919 "nhwc/reorg_layer.cl",
920#include "./cl_kernels/nhwc/reorg_layer.clembed"
921 },
922 {
923 "nhwc/scale.cl",
924#include "./cl_kernels/nhwc/scale.clembed"
925 },
926 {
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100927 "nhwc/space_to_batch.cl",
928#include "./cl_kernels/nhwc/space_to_batch.clembed"
929 },
930 {
931 "nhwc/space_to_depth.cl",
932#include "./cl_kernels/nhwc/space_to_depth.clembed"
933 },
934 {
935 "nhwc/winograd_filter_transform.cl",
936#include "./cl_kernels/nhwc/winograd_filter_transform.clembed"
937 },
938 {
939 "nhwc/winograd_input_transform.cl",
940#include "./cl_kernels/nhwc/winograd_input_transform.clembed"
941 },
Adnan AlSinan7075fe22021-07-05 13:12:52 +0100942 {
943 "nhwc/winograd_output_transform.cl",
944#include "./cl_kernels/nhwc/winograd_output_transform.clembed"
Georgios Pinitas908f6162021-05-04 10:11:09 +0100945 },
Adnan AlSinanf81f51c2021-07-26 18:18:37 +0100946#endif /* ENABLE_NHWC_KERNELS */
Georgios Pinitas908f6162021-05-04 10:11:09 +0100947#endif /* EMBEDDED_KERNELS */
948};
949
950ClKernelLibrary &ClKernelLibrary::get()
951{
952 static ClKernelLibrary _kernel_library;
953 return _kernel_library;
954}
955
956std::string ClKernelLibrary::program_name(const std::string &kernel_name) const
957{
958 // Find which program contains the kernel
959 auto kernel_program_it = _kernel_program_map.find(kernel_name);
960
961 if(_kernel_program_map.end() == kernel_program_it)
962 {
963 ARM_COMPUTE_ERROR_VAR("Kernel %s not found in the CLKernelLibrary", kernel_name.c_str());
964 }
965
966 const std::string program_name = kernel_program_it->second;
967
968 return program_name;
969}
970
971void ClKernelLibrary::set_kernel_path(std::string kernel_path)
972{
973 _kernel_path = std::move(kernel_path);
974 _kernel_path += "/";
975}
976
977const std::string &ClKernelLibrary::kernel_path() const
978{
979 return _kernel_path;
980}
981
982ClKernelLibrary::ClProgramInfo ClKernelLibrary::program(const std::string &program_name) const
983{
984#ifdef EMBEDDED_KERNELS
985#ifdef ARM_COMPUTE_COMPRESSED_KERNELS
986 const auto inflatted_program_source_it = _decompressed_source_map.find(program_name);
987 if(inflatted_program_source_it != _decompressed_source_map.end())
988 {
989 return ClProgramInfo{ inflatted_program_source_it->second, false };
990 }
991#endif /* ARM_COMPUTE_COMPRESSED_KERNELS */
992
993 const auto program_source_it = _program_source_map.find(program_name);
994 if(program_source_it == _program_source_map.end())
995 {
996 ARM_COMPUTE_ERROR_VAR("Embedded program for %s does not exist.", program_name.c_str());
997 }
998 std::string program_source = program_source_it->second;
999
1000#ifdef ARM_COMPUTE_COMPRESSED_KERNELS
1001 std::string decompressed_program_source = decompress_zlib(decode_base64(program_source_it->second));
1002 ARM_COMPUTE_ERROR_ON_MSG(decompressed_program_source.empty(), "Cannot de-compress requested program");
1003 _decompressed_source_map.insert(std::make_pair(program_name, decompressed_program_source));
1004 program_source = std::move(decompressed_program_source);
1005#endif /* ARM_COMPUTE_COMPRESSED_KERNELS */
1006
1007 return ClProgramInfo{ program_source, false };
1008#else /* EMBEDDED_KERNELS */
1009 // Check for binary
1010 std::string source_name = _kernel_path + program_name;
1011 std::string binary_name = source_name + "bin";
1012 std::string program_source{};
1013 bool is_binary = false;
1014
1015 if(std::ifstream(binary_name).is_open())
1016 {
1017 program_source = read_file(binary_name, true);
1018 is_binary = true;
1019 }
1020 else if(std::ifstream(source_name).is_open())
1021 {
1022 program_source = read_file(source_name, false);
1023 }
1024 else
1025 {
1026 ARM_COMPUTE_ERROR_VAR("Kernel file %s does not exist.", source_name.c_str());
1027 }
1028
1029 return ClProgramInfo{ program_source, is_binary };
1030#endif /* EMBEDDED_KERNELS */
1031}
1032} // namespace opencl
1033} // namespace arm_compute