blob: b878a2121fb9c33829cf501b0dbff16d94a1c6fc [file] [log] [blame]
steniu0127b386c2017-07-18 17:37:43 +01001/*
Georgios Pinitasf52cd782019-03-25 14:06:14 +00002 * Copyright (c) 2017-2019 ARM Limited.
steniu0127b386c2017-07-18 17:37:43 +01003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "arm_compute/core/CL/kernels/CLDirectConvolutionLayerKernel.h"
25
26#include "arm_compute/core/AccessWindowStatic.h"
27#include "arm_compute/core/CL/CLHelpers.h"
28#include "arm_compute/core/CL/CLKernelLibrary.h"
Vidhya Sudhan Loganathanf1f49062018-05-25 13:21:26 +010029#include "arm_compute/core/CL/CLValidate.h"
steniu0127b386c2017-07-18 17:37:43 +010030#include "arm_compute/core/CL/ICLTensor.h"
31#include "arm_compute/core/Error.h"
32#include "arm_compute/core/Helpers.h"
33#include "arm_compute/core/IAccessWindow.h"
34#include "arm_compute/core/ITensor.h"
35#include "arm_compute/core/Types.h"
Gian Marco Iodice5cb4d6a2017-08-08 10:53:00 +010036#include "arm_compute/core/Utils.h"
Giorgio Arenac0f54432018-03-16 14:02:34 +000037#include "arm_compute/core/utils/misc/ShapeCalculator.h"
Chunosovd621bca2017-11-03 17:33:15 +070038#include "arm_compute/core/utils/quantization/AsymmHelpers.h"
steniu0127b386c2017-07-18 17:37:43 +010039#include "support/ToolchainSupport.h"
40
41using namespace arm_compute;
42
Georgios Pinitas30902ed2017-11-14 15:32:57 +000043namespace
44{
Georgios Pinitas631c41a2017-12-06 11:53:03 +000045Status validate_arguments(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info)
Georgios Pinitas30902ed2017-11-14 15:32:57 +000046{
Vidhya Sudhan Loganathanf1f49062018-05-25 13:21:26 +010047 ARM_COMPUTE_RETURN_ERROR_ON_F16_UNSUPPORTED(input);
Vidhya Sudhan Loganathan7485d5a2018-07-04 09:34:00 +010048 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::F16, DataType::F32);
Georgios Pinitas30902ed2017-11-14 15:32:57 +000049 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
Pablo Tello3d319462018-06-21 15:13:17 +010050
51 const DataLayout data_layout = input->data_layout();
52 const int width_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
53 const int height_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
54 const int channel_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::CHANNEL);
55
56 ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->dimension(width_idx) != weights->dimension(height_idx), "Weights should have same width and height");
Michalis Spyrou45091732019-05-13 17:41:01 +010057 ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->dimension(width_idx) != 1 && weights->dimension(width_idx) != 3 && weights->dimension(width_idx) != 5 && weights->dimension(width_idx) != 9,
58 "Kernel sizes other than 1x1, 3x3, 5x5 or 9x9 are not supported");
Pablo Tello3d319462018-06-21 15:13:17 +010059 ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->dimension(channel_idx) != input->dimension(channel_idx),
Georgios Pinitas30902ed2017-11-14 15:32:57 +000060 "Weights feature map dimension should match the respective input's one");
Pablo Tello3d319462018-06-21 15:13:17 +010061 ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->num_dimensions() > 4, "Weights can be at most 4 dimensional");
62 ARM_COMPUTE_RETURN_ERROR_ON_MSG((weights->dimension(width_idx) == 1) && std::get<0>(conv_info.stride()) > 3, "Strides larger than 3 not supported for 1x1 convolution.");
63 ARM_COMPUTE_RETURN_ERROR_ON_MSG((weights->dimension(width_idx) == 3 || weights->dimension(width_idx) == 5) && std::get<0>(conv_info.stride()) > 2,
Georgios Pinitas30902ed2017-11-14 15:32:57 +000064 "Strides larger than 2 not supported for 3x3 convolution.");
Michalis Spyrou45091732019-05-13 17:41:01 +010065 ARM_COMPUTE_RETURN_ERROR_ON_MSG((weights->dimension(width_idx) == 9) && data_layout == DataLayout::NCHW, "Only NHWC layout is supported for 9x9 convolution.");
Georgios Pinitas30902ed2017-11-14 15:32:57 +000066
67 if(biases != nullptr)
68 {
69 if(is_data_type_quantized_asymmetric(input->data_type()))
70 {
71 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(biases, 1, DataType::S32);
72 }
73 else
74 {
75 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases);
76 }
77 ARM_COMPUTE_RETURN_ERROR_ON_MSG(biases->dimension(0) != weights->dimension(3),
78 "Biases size and number of input feature maps should match");
79 ARM_COMPUTE_RETURN_ERROR_ON_MSG(biases->num_dimensions() > 1,
80 "Biases should be one dimensional");
81 }
82
83 // Checks performed when output is configured
84 if(output->total_size() != 0)
85 {
86 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(output->tensor_shape(),
Giorgio Arenac0f54432018-03-16 14:02:34 +000087 misc::shape_calculator::compute_deep_convolution_shape(*input, *weights, conv_info));
Georgios Pinitas30902ed2017-11-14 15:32:57 +000088 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
Georgios Pinitas30902ed2017-11-14 15:32:57 +000089 }
90
Georgios Pinitas631c41a2017-12-06 11:53:03 +000091 return Status{};
Georgios Pinitas30902ed2017-11-14 15:32:57 +000092}
93
Pablo Tello3d319462018-06-21 15:13:17 +010094inline bool can_run_optimized_kernel_for_bifrost(GPUTarget gpu_target, unsigned int conv_stride_x, unsigned int conv_stride_y, unsigned int kernel_size,
95 DataType data_type, DataLayout data_layout)
Giorgio Arena59486342017-12-01 10:42:47 +000096{
Georgios Pinitasa34286e2018-09-04 12:18:50 +010097 return gpu_target_is_in(gpu_target,
98 GPUTarget::G71, GPUTarget::G72, GPUTarget::G76,
99 GPUTarget::G51, GPUTarget::G51BIG, GPUTarget::G51LIT,
100 GPUTarget::G52, GPUTarget::G52LIT)
101 && (kernel_size <= 5)
102 && (conv_stride_x == 1) && (conv_stride_y == 1)
103 && (data_type == DataType::F32)
104 && (data_layout == DataLayout::NCHW);
Pablo Tello3d319462018-06-21 15:13:17 +0100105}
Giorgio Arena59486342017-12-01 10:42:47 +0000106
Michalis Spyrou45091732019-05-13 17:41:01 +0100107inline bool can_run_optimized_kernel_for_bifrost_nhwc(GPUTarget gpu_target, unsigned int conv_stride_x, unsigned int conv_stride_y, unsigned int kernel_size,
108 DataType data_type, DataLayout data_layout)
109{
110 return gpu_target_is_in(gpu_target,
111 GPUTarget::G71, GPUTarget::G72, GPUTarget::G76,
112 GPUTarget::G51, GPUTarget::G51BIG, GPUTarget::G51LIT,
113 GPUTarget::G52, GPUTarget::G52LIT)
114 && (kernel_size == 9)
115 && (conv_stride_x == 1) && (conv_stride_y == 1)
116 && (data_type == DataType::F32)
117 && (data_layout == DataLayout::NHWC);
118}
119
Pablo Tello3d319462018-06-21 15:13:17 +0100120inline void setup_num_elems(unsigned int &num_elems_read_per_iteration_x, unsigned int &num_elems_read_per_iteration_y,
121 unsigned int &num_elems_written_per_iteration_x, unsigned int &num_elems_written_per_iteration_y,
122 unsigned int kernel_size, const PadStrideInfo &conv_info, const GPUTarget target, ITensorInfo *input)
123{
124 const DataType data_type = input->data_type();
125 const DataLayout data_layout = input->data_layout();
126 unsigned int conv_stride_x = std::get<0>(conv_info.stride());
127 unsigned int conv_stride_y = std::get<1>(conv_info.stride());
Giorgio Arena59486342017-12-01 10:42:47 +0000128
Pablo Tello3d319462018-06-21 15:13:17 +0100129 const bool run_optimized_bifrost = can_run_optimized_kernel_for_bifrost(target, conv_stride_x, conv_stride_y, kernel_size, data_type, data_layout);
Giorgio Arena59486342017-12-01 10:42:47 +0000130
Pablo Tello3d319462018-06-21 15:13:17 +0100131 if(run_optimized_bifrost)
Giorgio Arena59486342017-12-01 10:42:47 +0000132 {
133 // Configure kernel window
Giorgio Arena59486342017-12-01 10:42:47 +0000134 switch(kernel_size)
135 {
136 case 1:
137 {
138 num_elems_read_per_iteration_x = 4;
139 num_elems_read_per_iteration_y = 4;
140 num_elems_written_per_iteration_x = 4;
141 num_elems_written_per_iteration_y = 4;
142 break;
143 }
144 case 3:
145 {
146 num_elems_read_per_iteration_x = 6;
147 num_elems_read_per_iteration_y = 5;
148 num_elems_written_per_iteration_x = 4;
149 num_elems_written_per_iteration_y = 3;
150 break;
151 }
152 case 5:
153 {
154 num_elems_read_per_iteration_x = 8;
155 num_elems_read_per_iteration_y = 6;
156 num_elems_written_per_iteration_x = 4;
157 num_elems_written_per_iteration_y = 2;
158 break;
159 }
160 default:
161 {
162 ARM_COMPUTE_ERROR("Kernel size not optimized for Bifrost");
163 }
164 }
165 }
Michalis Spyrou45091732019-05-13 17:41:01 +0100166 else if(data_layout == DataLayout::NCHW)
Giorgio Arena59486342017-12-01 10:42:47 +0000167 {
Giorgio Arena59486342017-12-01 10:42:47 +0000168 num_elems_read_per_iteration_y = kernel_size;
169 num_elems_written_per_iteration_x = 8;
170 num_elems_written_per_iteration_y = 1;
Anthony Barbiercc9fed52017-12-13 10:46:00 +0000171 switch(kernel_size)
172 {
173 case 1:
174 switch(conv_stride_x)
175 {
176 case 1:
177 num_elems_read_per_iteration_x = 8;
178 break;
179 case 2:
180 num_elems_read_per_iteration_x = 16;
181 break;
182 case 3:
183 switch(input->element_size())
184 {
185 case 1:
186 num_elems_read_per_iteration_x = 28;
187 break;
188 case 2:
189 num_elems_read_per_iteration_x = 24;
190 break;
191 case 4:
192 num_elems_read_per_iteration_x = 22;
193 break;
194 default:
195 ARM_COMPUTE_ERROR("Invalid data size");
196 }
197 break;
198 default:
199 ARM_COMPUTE_ERROR("Invalid convolution stride X");
200 }
201 break;
202 case 3:
203 switch(conv_stride_x)
204 {
205 case 1:
206 num_elems_read_per_iteration_x = 10;
207 break;
208 case 2:
209 num_elems_read_per_iteration_x = 17;
210 break;
211 default:
212 ARM_COMPUTE_ERROR("Invalid convolution stride X");
213 }
214 break;
215 case 5:
216 switch(conv_stride_x)
217 {
218 case 1:
219 num_elems_read_per_iteration_x = 12;
220 break;
221 case 2:
222 num_elems_read_per_iteration_x = 20;
223 break;
224 default:
225 ARM_COMPUTE_ERROR("Invalid convolution stride X");
226 }
227 break;
228 default:
229 ARM_COMPUTE_ERROR("Invalid direct convolution size");
230 }
Giorgio Arena59486342017-12-01 10:42:47 +0000231 }
Michalis Spyrou45091732019-05-13 17:41:01 +0100232 else // data_layout == NHWC
Pablo Tello3d319462018-06-21 15:13:17 +0100233 {
Michalis Spyrou45091732019-05-13 17:41:01 +0100234 const bool run_optimized_bifrost_nhwc = can_run_optimized_kernel_for_bifrost_nhwc(target, conv_stride_x, conv_stride_y, kernel_size, data_type, data_layout);
235
Pablo Tello3d319462018-06-21 15:13:17 +0100236 num_elems_written_per_iteration_x = 1;
Michalis Spyrou45091732019-05-13 17:41:01 +0100237
238 if(run_optimized_bifrost_nhwc)
239 {
240 num_elems_read_per_iteration_x = 4;
241 }
242
Pablo Tello3d319462018-06-21 15:13:17 +0100243 switch(kernel_size)
244 {
245 case 1:
246 switch(conv_stride_x)
247 {
248 case 1:
249 num_elems_read_per_iteration_y = 8;
250 num_elems_written_per_iteration_y = 8;
251 break;
252 case 2:
253 num_elems_read_per_iteration_y = 16;
254 num_elems_written_per_iteration_y = 8;
255 break;
256 default:
257 ARM_COMPUTE_ERROR("Invalid convolution stride X");
258 }
259 break;
260 case 3:
261 switch(conv_stride_x)
262 {
263 case 1:
264 num_elems_read_per_iteration_y = 10;
265 num_elems_written_per_iteration_y = 8;
266 break;
267 case 2:
268 num_elems_read_per_iteration_y = 17;
269 num_elems_written_per_iteration_y = 8;
270 break;
271 default:
272 ARM_COMPUTE_ERROR("Invalid convolution stride X");
273 }
274 break;
275 case 5:
276 switch(conv_stride_x)
277 {
278 case 1:
279 num_elems_read_per_iteration_y = 12;
280 num_elems_written_per_iteration_y = 8;
281 break;
282 case 2:
283 num_elems_read_per_iteration_y = 20;
284 num_elems_written_per_iteration_y = 8;
285 break;
286 default:
287 ARM_COMPUTE_ERROR("Invalid convolution stride X");
288 }
289 break;
Michalis Spyrou45091732019-05-13 17:41:01 +0100290 case 9:
291 switch(conv_stride_x)
292 {
293 case 1:
294 num_elems_read_per_iteration_y = 16;
295 num_elems_written_per_iteration_y = 8;
296 break;
297 case 2:
298 num_elems_read_per_iteration_y = 24;
299 num_elems_written_per_iteration_y = 8;
300 break;
301 default:
302 ARM_COMPUTE_ERROR("Invalid convolution stride X");
303 }
304 break;
Pablo Tello3d319462018-06-21 15:13:17 +0100305 default:
306 ARM_COMPUTE_ERROR("Not implemented.");
307 break;
308 }
309 }
310}
311
312std::pair<Status, Window> validate_and_configure_window(ITensorInfo *input, ITensorInfo *weights, ITensorInfo *output, const PadStrideInfo &conv_info, const GPUTarget target)
313{
314 const DataLayout data_layout = input->data_layout();
315 const int width_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
316 const unsigned int kernel_size = weights->dimension(width_idx);
317
318 // Get convolved dimensions
319 TensorShape output_shape = misc::shape_calculator::compute_deep_convolution_shape(*input, *weights, conv_info);
320
321 // Output auto inizialitation if not yet initialized
Georgios Pinitasf52cd782019-03-25 14:06:14 +0000322 // TODO(COMPMID-2078): input->clone()->set_tensor_shape(output_shape) doesn't work with subtensors for grouped direct convolutions (AlexNet).
Pablo Tello3d319462018-06-21 15:13:17 +0100323 auto_init_if_empty(*output, output_shape,
324 1,
325 input->data_type(),
326 input->quantization_info());
327
328 unsigned int num_elems_read_per_iteration_x = 0;
329 unsigned int num_elems_read_per_iteration_y = 0;
330 unsigned int num_elems_written_per_iteration_x = 0;
331 unsigned int num_elems_written_per_iteration_y = 0;
332
333 unsigned int conv_pad_left = conv_info.pad_left();
334 unsigned int conv_pad_top = conv_info.pad_top();
335 unsigned int conv_stride_x = std::get<0>(conv_info.stride());
336 unsigned int conv_stride_y = std::get<1>(conv_info.stride());
337
338 setup_num_elems(num_elems_read_per_iteration_x, num_elems_read_per_iteration_y,
339 num_elems_written_per_iteration_x, num_elems_written_per_iteration_y,
340 kernel_size, conv_info, target, input);
341
Giorgio Arena59486342017-12-01 10:42:47 +0000342 // Create window and update padding
Anthony Barbiercc9fed52017-12-13 10:46:00 +0000343 bool window_changed = false;
344 Window win = calculate_max_window(*output, Steps(num_elems_written_per_iteration_x, num_elems_written_per_iteration_y));
Giorgio Arena59486342017-12-01 10:42:47 +0000345
Pablo Tello3d319462018-06-21 15:13:17 +0100346 if(data_layout == DataLayout::NHWC)
347 {
348 AccessWindowStatic input_access(input, 0, -conv_pad_left,
349 num_elems_read_per_iteration_x,
350 ceil_to_multiple(input->dimension(1) + conv_info.pad_right(), num_elems_read_per_iteration_y));
351 AccessWindowStatic weights_access(weights, 0, 0, weights->dimension(0), weights->dimension(1));
352 AccessWindowRectangle output_access(output, 0, 0, num_elems_written_per_iteration_x, num_elems_written_per_iteration_y);
353 window_changed = update_window_and_padding(win, input_access, weights_access, output_access);
354 output_access.set_valid_region(win, ValidRegion(Coordinates(), output->tensor_shape()));
355 Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};
356 return std::make_pair(err, win);
357 }
358 else if(data_layout == DataLayout::NCHW)
359 {
360 AccessWindowRectangle input_access(input, -conv_pad_left, -conv_pad_top, num_elems_read_per_iteration_x, num_elems_read_per_iteration_y, conv_stride_x, conv_stride_y);
361 AccessWindowStatic weights_access(weights, 0, 0, kernel_size, kernel_size);
362 AccessWindowRectangle output_access(output, 0, 0, num_elems_written_per_iteration_x, num_elems_written_per_iteration_y);
363 window_changed = update_window_and_padding(win, input_access, weights_access, output_access);
364 output_access.set_valid_region(win, ValidRegion(Coordinates(), output->tensor_shape()));
365 Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};
366 return std::make_pair(err, win);
367 }
368 else
369 {
370 ARM_COMPUTE_ERROR("Not supported");
371 }
Giorgio Arena59486342017-12-01 10:42:47 +0000372}
373} // namespace
374
375CLDirectConvolutionLayerKernel::CLDirectConvolutionLayerKernel()
376 : _input(nullptr), _biases(nullptr), _weights(nullptr), _output(nullptr), _border_size(0), _conv_stride_x(0), _conv_stride_y(0)
377{
378}
379
380BorderSize CLDirectConvolutionLayerKernel::border_size() const
381{
382 return _border_size;
383}
384
385void CLDirectConvolutionLayerKernel::configure(const ICLTensor *input, const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, const PadStrideInfo &conv_info)
386{
387 ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
388
Pablo Tello3d319462018-06-21 15:13:17 +0100389 const DataLayout data_layout = input->info()->data_layout();
390 const int width_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
391 const int height_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
392 const int channel_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::CHANNEL);
393
394 const unsigned int kernel_size = weights->info()->dimension(width_idx);
Giorgio Arena59486342017-12-01 10:42:47 +0000395 const DataType data_type = input->info()->data_type();
396
397 // Get convolved dimensions
Giorgio Arenac0f54432018-03-16 14:02:34 +0000398 TensorShape output_shape = misc::shape_calculator::compute_deep_convolution_shape(*input->info(), *weights->info(), conv_info);
Giorgio Arena59486342017-12-01 10:42:47 +0000399
400 // Output auto inizialitation if not yet initialized
Georgios Pinitasf52cd782019-03-25 14:06:14 +0000401 // TODO(COMPMID-2078): input->clone()->set_tensor_shape(output_shape) doesn't work with subtensors for grouped direct convolutions (AlexNet).
Giorgio Arena59486342017-12-01 10:42:47 +0000402 auto_init_if_empty(*output->info(),
403 output_shape,
404 1,
405 input->info()->data_type(),
Giorgio Arena59486342017-12-01 10:42:47 +0000406 input->info()->quantization_info());
407
408 // Perform validation step
409 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(),
410 weights->info(),
411 (biases != nullptr) ? biases->info() : nullptr,
412 output->info(),
413 conv_info));
414
415 _conv_stride_x = std::get<0>(conv_info.stride());
416 _conv_stride_y = std::get<1>(conv_info.stride());
Pablo Tello3d319462018-06-21 15:13:17 +0100417
418 if(data_layout == DataLayout::NHWC)
419 {
420 _border_size = BorderSize(conv_info.pad_left(), 0, conv_info.pad_right(), 0);
421 }
422 else if(data_layout == DataLayout::NCHW)
423 {
424 _border_size = BorderSize(conv_info.pad_top(), conv_info.pad_right(), conv_info.pad_bottom(), conv_info.pad_left());
425 }
426 else
427 {
428 ARM_COMPUTE_ERROR("Not supported");
429 }
Giorgio Arena59486342017-12-01 10:42:47 +0000430
431 _input = input;
432 _weights = weights;
433 _output = output;
434 _biases = biases;
435
Michalis Spyroua9676112018-02-22 18:07:43 +0000436 const GPUTarget gpu_target = get_target();
Giorgio Arena59486342017-12-01 10:42:47 +0000437
438 std::stringstream kernel_name;
439 kernel_name << "direct_convolution" << kernel_size << "x" << kernel_size;
Pablo Tello3d319462018-06-21 15:13:17 +0100440 if(data_layout == DataLayout::NHWC)
441 {
442 kernel_name << "_" << lower_string(string_from_data_layout(data_layout));
443 }
Giorgio Arena59486342017-12-01 10:42:47 +0000444
445 CLBuildOptions build_options;
446 build_options.add_option_if(_biases != nullptr, std::string("-DHAS_BIAS"));
447
Pablo Tello3d319462018-06-21 15:13:17 +0100448 const bool run_optimized_for_bifrost = can_run_optimized_kernel_for_bifrost(gpu_target, _conv_stride_x, _conv_stride_y, kernel_size, data_type, data_layout);
449
450 if(run_optimized_for_bifrost)
Giorgio Arena59486342017-12-01 10:42:47 +0000451 {
Pablo Tello3d319462018-06-21 15:13:17 +0100452 build_options.add_option(std::string("-DWEIGHTS_DEPTH=" + support::cpp11::to_string(_weights->info()->dimension(channel_idx))));
Giorgio Arena59486342017-12-01 10:42:47 +0000453
454 kernel_name << "_f32_bifrost";
455 _kernel = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel(kernel_name.str(), build_options.options()));
Giorgio Arena59486342017-12-01 10:42:47 +0000456 }
457 else
458 {
Pablo Tellod041a832018-10-03 17:11:09 +0100459 const bool is_quantized_asymm = is_data_type_quantized_asymmetric(data_type);
Giorgio Arena59486342017-12-01 10:42:47 +0000460 build_options.add_option_if(is_quantized_asymm, std::string("-DKERNEL_SIZE=" + support::cpp11::to_string(kernel_size)));
461 build_options.add_option(std::string("-DDATA_TYPE=" + get_cl_type_from_data_type(data_type)));
462 build_options.add_option(std::string("-DDATA_SIZE=" + get_data_size_from_data_type(data_type)));
Pablo Tello3d319462018-06-21 15:13:17 +0100463 build_options.add_option(std::string("-DWEIGHTS_DEPTH=" + support::cpp11::to_string(_weights->info()->dimension(channel_idx))));
Giorgio Arena59486342017-12-01 10:42:47 +0000464 build_options.add_option(std::string("-DSTRIDE_X=" + support::cpp11::to_string(_conv_stride_x)));
Pablo Tello3d319462018-06-21 15:13:17 +0100465 if(data_layout == DataLayout::NHWC)
466 {
Michalis Spyrou45091732019-05-13 17:41:01 +0100467 const bool run_optimized_for_bifrost_nhwc = can_run_optimized_kernel_for_bifrost_nhwc(gpu_target, _conv_stride_x, _conv_stride_y, kernel_size, data_type, data_layout);
Pablo Tello3d319462018-06-21 15:13:17 +0100468 build_options.add_option(std::string("-DDATA_LAYOUT_NHWC=1"));
469 build_options.add_option(std::string("-DDST_HEIGHT=" + support::cpp11::to_string(_output->info()->dimension(height_idx))));
470 build_options.add_option(std::string("-DDST_WIDTH=" + support::cpp11::to_string(_output->info()->dimension(width_idx))));
471 build_options.add_option(std::string("-DSRC_HEIGHT=" + support::cpp11::to_string(_input->info()->dimension(height_idx))));
472 build_options.add_option(std::string("-DSRC_WIDTH=" + support::cpp11::to_string(_input->info()->dimension(width_idx))));
473 build_options.add_option(std::string("-DPAD_LEFT=" + support::cpp11::to_string(conv_info.pad_left())));
474 build_options.add_option(std::string("-DPAD_TOP=" + support::cpp11::to_string(conv_info.pad_top())));
475 build_options.add_option(std::string("-DSTRIDE_Y=" + support::cpp11::to_string(_conv_stride_y)));
Michalis Spyrou45091732019-05-13 17:41:01 +0100476 if(run_optimized_for_bifrost_nhwc)
477 {
478 const unsigned int num_elems_read_per_iteration_x = 4;
479 _border_size.right = num_elems_read_per_iteration_x;
480 build_options.add_option("-DVEC_SIZE=" + support::cpp11::to_string(num_elems_read_per_iteration_x));
481 }
Pablo Tello3d319462018-06-21 15:13:17 +0100482 }
Vidhya Sudhan Loganathan7485d5a2018-07-04 09:34:00 +0100483 build_options.add_option(std::string("-DDATA_TYPE_PROMOTED=" + get_cl_type_from_data_type(data_type)));
Giorgio Arena59486342017-12-01 10:42:47 +0000484 // Create kernel
485 _kernel = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel(is_quantized_asymm ? "direct_convolution_1x1_3x3_5x5_quantized" : kernel_name.str(),
486 build_options.options()));
487 }
488
489 // Configure kernel window
490 auto win_config = validate_and_configure_window(input->info(), weights->info(), output->info(), conv_info, gpu_target);
491 ARM_COMPUTE_ERROR_THROW_ON(win_config.first);
Anthony Barbierb6eb3532018-08-08 13:20:04 +0100492 ICLKernel::configure_internal(win_config.second);
Giorgio Arena59486342017-12-01 10:42:47 +0000493
494 // Set static kernel arguments
495 if(is_data_type_quantized_asymmetric(data_type))
496 {
Georgios Pinitas4c5469b2019-05-21 13:32:43 +0100497 const UniformQuantizationInfo iqinfo = _input->info()->quantization_info().uniform();
498 const UniformQuantizationInfo wqinfo = _weights->info()->quantization_info().uniform();
499 const UniformQuantizationInfo oqinfo = _output->info()->quantization_info().uniform();
500
Giorgio Arena59486342017-12-01 10:42:47 +0000501 int output_multiplier = 0;
502 int output_shift = 0;
503
Georgios Pinitas4c5469b2019-05-21 13:32:43 +0100504 float multiplier = iqinfo.scale * wqinfo.scale / oqinfo.scale;
Giorgio Arena59486342017-12-01 10:42:47 +0000505 ARM_COMPUTE_THROW_ON_ERROR(quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier, &output_shift));
506
507 unsigned int idx = 3 * num_arguments_per_3D_tensor() + ((_biases != nullptr) ? num_arguments_per_1D_tensor() : 0) + 1;
Georgios Pinitas4c5469b2019-05-21 13:32:43 +0100508 _kernel.setArg(idx++, -iqinfo.offset);
509 _kernel.setArg(idx++, -wqinfo.offset);
510 _kernel.setArg(idx++, oqinfo.offset);
Giorgio Arena59486342017-12-01 10:42:47 +0000511 _kernel.setArg(idx++, output_multiplier);
512 _kernel.setArg(idx++, output_shift);
513 }
514
515 // Set config_id for enabling LWS tuning
516 _config_id = "direct_convolution_";
517 _config_id += lower_string(string_from_data_type(data_type));
518 _config_id += "_";
519 _config_id += support::cpp11::to_string(kernel_size);
520 _config_id += "_";
Georgios Pinitas15997872018-02-19 13:58:22 +0000521 _config_id += support::cpp11::to_string(border_size().left);
Giorgio Arena59486342017-12-01 10:42:47 +0000522 _config_id += "_";
Georgios Pinitas15997872018-02-19 13:58:22 +0000523 _config_id += support::cpp11::to_string(border_size().top);
Giorgio Arena59486342017-12-01 10:42:47 +0000524 _config_id += "_";
Georgios Pinitas15997872018-02-19 13:58:22 +0000525 _config_id += support::cpp11::to_string(border_size().right);
Giorgio Arena59486342017-12-01 10:42:47 +0000526 _config_id += "_";
Georgios Pinitas15997872018-02-19 13:58:22 +0000527 _config_id += support::cpp11::to_string(border_size().bottom);
Giorgio Arena59486342017-12-01 10:42:47 +0000528 _config_id += "_";
529 _config_id += support::cpp11::to_string(_conv_stride_x);
530 _config_id += "_";
531 _config_id += support::cpp11::to_string(_conv_stride_y);
532 _config_id += "_";
Pablo Tello3d319462018-06-21 15:13:17 +0100533 _config_id += support::cpp11::to_string(output->info()->dimension(width_idx));
Giorgio Arena59486342017-12-01 10:42:47 +0000534 _config_id += "_";
Pablo Tello3d319462018-06-21 15:13:17 +0100535 _config_id += support::cpp11::to_string(output->info()->dimension(height_idx));
536 _config_id += "_";
537 _config_id += lower_string(string_from_data_layout(data_layout));
Giorgio Arena59486342017-12-01 10:42:47 +0000538}
539
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000540Status CLDirectConvolutionLayerKernel::validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info,
541 const GPUTarget target)
Giorgio Arena59486342017-12-01 10:42:47 +0000542{
543 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, weights, biases, output, conv_info));
544 ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(input->clone().get(), weights->clone().get(), output->clone().get(), conv_info, target).first);
545
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000546 return Status{};
Giorgio Arena59486342017-12-01 10:42:47 +0000547}
548
SiCong Lic51b72f2017-07-28 14:46:20 +0100549void CLDirectConvolutionLayerKernel::run(const Window &window, cl::CommandQueue &queue)
steniu0127b386c2017-07-18 17:37:43 +0100550{
551 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
552 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(IKernel::window(), window);
553
554 // Get initial windows
555 Window slice = window.first_slice_window_3D();
556 Window win_in = window;
557
Jaroslaw Rzepecki2ecbada2017-11-29 13:51:34 +0000558 win_in.adjust(Window::DimX, -_border_size.left, true);
559 win_in.adjust(Window::DimY, -_border_size.top, true);
steniu0127b386c2017-07-18 17:37:43 +0100560
Pablo Tello3d319462018-06-21 15:13:17 +0100561 const DataLayout data_layout = _input->info()->data_layout();
562 const int width_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
563 const int height_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
steniu0127b386c2017-07-18 17:37:43 +0100564
Pablo Tello3d319462018-06-21 15:13:17 +0100565 win_in.set_dimension_step(width_idx, window[width_idx].step() * _conv_stride_x);
566 win_in.set_dimension_step(height_idx, window[height_idx].step() * _conv_stride_y);
567
568 Window slice_in = win_in.first_slice_window_3D();
569 unsigned int idx1 = 2 * num_arguments_per_3D_tensor();
steniu0127b386c2017-07-18 17:37:43 +0100570 add_3D_tensor_argument(idx1, _weights, slice);
571
572 if(_biases != nullptr)
573 {
574 Window slice_biases;
SiCong Li86b53332017-08-23 11:02:43 +0100575 slice_biases.use_tensor_dimensions(_biases->info()->tensor_shape());
steniu0127b386c2017-07-18 17:37:43 +0100576 add_1D_tensor_argument(idx1, _biases, slice_biases);
577 }
578
Gian Marco Iodice5cb4d6a2017-08-08 10:53:00 +0100579 _kernel.setArg(idx1++, static_cast<unsigned int>(_weights->info()->strides_in_bytes()[3]));
580
steniu0127b386c2017-07-18 17:37:43 +0100581 do
582 {
583 unsigned int idx = 0;
584 add_3D_tensor_argument(idx, _input, slice_in);
585 add_3D_tensor_argument(idx, _output, slice);
Anthony Barbierb6eb3532018-08-08 13:20:04 +0100586 enqueue(queue, *this, slice, lws_hint());
steniu0127b386c2017-07-18 17:37:43 +0100587 }
588 while(window.slide_window_slice_3D(slice) && win_in.slide_window_slice_3D(slice_in));
589}