blob: d7a1778e266f7dac5d44436ee4859c5ddbcf4d81 [file] [log] [blame]
Anthony Barbier6ff3b192017-09-04 18:44:23 +01001/*
Matthew Bentham758b5ba2020-03-05 23:37:48 +00002 * Copyright (c) 2017-2020 ARM Limited.
Anthony Barbier6ff3b192017-09-04 18:44:23 +01003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "arm_compute/core/CL/kernels/CLSoftmaxLayerKernel.h"
25
26#include "arm_compute/core/AccessWindowStatic.h"
27#include "arm_compute/core/CL/CLHelpers.h"
28#include "arm_compute/core/CL/CLKernelLibrary.h"
Vidhya Sudhan Loganathanf1f49062018-05-25 13:21:26 +010029#include "arm_compute/core/CL/CLValidate.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010030#include "arm_compute/core/CL/ICLTensor.h"
31#include "arm_compute/core/CL/OpenCL.h"
32#include "arm_compute/core/Helpers.h"
Sang-Hoon Park62eeb532019-10-29 13:13:19 +000033#include "arm_compute/core/KernelDescriptors.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010034#include "arm_compute/core/TensorInfo.h"
35#include "arm_compute/core/Utils.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010036#include "arm_compute/core/Window.h"
Chunosovf450caa2017-11-08 16:09:35 +070037#include "arm_compute/core/utils/quantization/AsymmHelpers.h"
Matthew Bentham758b5ba2020-03-05 23:37:48 +000038#include "support/StringSupport.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010039
40#include <set>
41#include <string>
42
43using namespace arm_compute;
Georgios Pinitas30902ed2017-11-14 15:32:57 +000044
Chunosovf450caa2017-11-08 16:09:35 +070045namespace
46{
47/** Calculates softmax parameters from the quantized input scale and scaling factor for the exponent and places them as build options.
48 *
49 * Prepares these build options:
50 * -INPUT_BETA_MULTIPLIER, INPUT_BETA_LEFT_SHIFT - quantized representation of beta multiplier.
51 * -DIFF_MIN - threshold difference between maximum value of input data and current processed value,
52 * it defines whether the value will be taken into account or not.
53 *
54 * @param[in] build_opts Build options to extend
55 * @param[in] input_scale Input scaling factor
56 * @param[in] beta Exponent scaling factor beta
57 */
58CLBuildOptions prepare_quantized_softmax_build_options(float input_scale, float beta)
59{
60 // Number of integer bits in temporary fixed-point representation of current-to-max difference
61 static const int scaled_diff_int_bits = 5;
62 // Number of integer bits used in temporary fixed-point representation of exponent accumulator
63 static const int exp_accumulation_in_bits = 12;
64
65 const double beta_multiplier = std::min(
66 1.0 * beta * input_scale * (1 << (31 - scaled_diff_int_bits)),
Michalis Spyroua4f378d2019-04-26 14:54:54 +010067 (1LL << 31) - 1.0);
68 int input_beta_multiplier;
69 int input_beta_left_shift;
Chunosovf450caa2017-11-08 16:09:35 +070070 quantization::calculate_quantized_multiplier_greater_than_one(beta_multiplier, &input_beta_multiplier, &input_beta_left_shift);
71
Michalis Spyroua4f378d2019-04-26 14:54:54 +010072 const double max_input_rescaled = 1.0 * ((1 << scaled_diff_int_bits) - 1) * (1LL << (31 - scaled_diff_int_bits)) / (1LL << input_beta_left_shift);
Chunosovf450caa2017-11-08 16:09:35 +070073 const int diff_min = -1.f * std::floor(max_input_rescaled);
74
75 CLBuildOptions build_opts;
76 build_opts.add_option("-DSCALED_DIFF_INT_BITS=" + support::cpp11::to_string(scaled_diff_int_bits));
77 build_opts.add_option("-DEXP_ACCUMULATION_INT_BITS=" + support::cpp11::to_string(exp_accumulation_in_bits));
78 build_opts.add_option("-DINPUT_BETA_MULTIPLIER=" + support::cpp11::to_string(input_beta_multiplier));
79 build_opts.add_option("-DINPUT_BETA_LEFT_SHIFT=" + support::cpp11::to_string(input_beta_left_shift));
80 build_opts.add_option("-DDIFF_MIN=" + support::cpp11::to_string(diff_min));
81
82 return build_opts;
83}
Anthony Barbier6ff3b192017-09-04 18:44:23 +010084
Giorgio Arena4402cb92018-02-15 13:37:40 +000085Status validate_arguments_1DMaxShiftExpSum(const ITensorInfo *input, const ITensorInfo *max, const ITensorInfo *output, const ITensorInfo *sum)
Georgios Pinitas30902ed2017-11-14 15:32:57 +000086{
Vidhya Sudhan Loganathanf1f49062018-05-25 13:21:26 +010087 ARM_COMPUTE_RETURN_ERROR_ON_F16_UNSUPPORTED(input);
Sang-Hoon Park0779fec2019-11-13 17:08:12 +000088 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::F16, DataType::F32);
Georgios Pinitas30902ed2017-11-14 15:32:57 +000089 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(max, sum, output);
90
Giorgio Arena4402cb92018-02-15 13:37:40 +000091 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, max);
Giorgio Arena4402cb92018-02-15 13:37:40 +000092
Georgios Pinitas30902ed2017-11-14 15:32:57 +000093 const bool is_quantized_asymmetric = is_data_type_quantized_asymmetric(input->data_type());
94
95 // Checks performed when output is configured
96 if(output->total_size() != 0)
97 {
98 if(is_quantized_asymmetric)
99 {
100 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::S32);
101 }
102 else
103 {
104 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
105 }
106 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input, output);
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000107 }
108
109 // Checks performed when sum is configured
110 if(sum->total_size() != 0)
111 {
112 if(is_quantized_asymmetric)
113 {
114 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(sum, 1, DataType::S32);
115 }
116 else
117 {
118 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(max, sum);
119 }
120 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(max, sum);
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000121 }
122
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000123 return Status{};
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000124}
125
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000126Status validate_arguments_1DNorm(const ITensorInfo *input, const ITensorInfo *sum, const ITensorInfo *output, const SoftmaxKernelInfo &info)
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000127{
Vidhya Sudhan Loganathanf1f49062018-05-25 13:21:26 +0100128 ARM_COMPUTE_RETURN_ERROR_ON_F16_UNSUPPORTED(input);
Vidhya Sudhan Loganathan7485d5a2018-07-04 09:34:00 +0100129 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::S32, DataType::F16, DataType::F32);
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000130 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(sum, output);
131 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, sum);
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000132
133 // Note: output should always have a scale of 1/256 and offset 0
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000134 const QuantizationInfo allowed_quantization_info = get_softmax_output_quantization_info(info.input_data_type, info.is_log);
135 const bool is_quantized_asymmetric = is_data_type_quantized_asymmetric(info.input_data_type);
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000136
137 // Checks performed when output is configured
138 if(output->total_size() != 0)
139 {
140 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input, output);
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000141 if(!is_quantized_asymmetric)
142 {
143 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
144 }
145 else
146 {
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000147 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED);
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000148 ARM_COMPUTE_RETURN_ERROR_ON(output->quantization_info() != allowed_quantization_info);
149 }
150 }
151
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000152 return Status{};
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000153}
154
155// Window validation
156
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000157std::pair<Status, Window> validate_and_configure_window_1DMaxShiftExpSum(ITensorInfo *input, ITensorInfo *max, ITensorInfo *output, ITensorInfo *sum)
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000158{
159 // Output auto initialization if not yet initialized
160 auto_init_if_empty(*sum, input->clone()->set_tensor_shape(max->tensor_shape()));
161 auto_init_if_empty(*output, *input->clone());
162
163 CLLogits1DMaxShiftExpSumKernel::ParallelReductionInfo parallel_reduction_info = CLLogits1DMaxShiftExpSumKernel::is_parallel_reduction(input->dimension(0));
164 unsigned int vector_size = std::get<1>(parallel_reduction_info);
165 const unsigned int num_elems_x = ceil_to_multiple(input->tensor_shape().x(), vector_size);
166 Window win = calculate_max_window(*input, Steps(num_elems_x));
167
168 AccessWindowHorizontal input_access(input, 0, num_elems_x);
169 AccessWindowHorizontal max_access(max, 0, 1);
170 AccessWindowHorizontal output_access(output, 0, num_elems_x);
171 AccessWindowHorizontal sum_access(sum, 0, 1);
172
173 bool window_changed = update_window_and_padding(win, input_access, max_access, output_access, sum_access);
174
175 output_access.set_valid_region(win, input->valid_region());
176 sum_access.set_valid_region(win, ValidRegion(Coordinates(), sum->tensor_shape()));
177
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000178 Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000179 return std::make_pair(err, win);
180}
181
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000182std::pair<Status, Window> validate_and_configure_window_1DNorm(ITensorInfo *input, ITensorInfo *output, ITensorInfo *sum, const SoftmaxKernelInfo &info)
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000183{
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000184 const DataType output_data_type = info.input_data_type;
185 const QuantizationInfo allowed_quantization_info = get_softmax_output_quantization_info(info.input_data_type, info.is_log);
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000186
187 // Output auto initialization if not yet initialized
188 auto_init_if_empty(*output,
189 input->clone()->set_data_type(output_data_type).set_quantization_info(allowed_quantization_info));
190
191 constexpr unsigned int num_elems_processed_per_iteration = 16;
192
193 Window win = calculate_max_window(*input, Steps(num_elems_processed_per_iteration));
194
195 AccessWindowHorizontal input_access(input, 0, num_elems_processed_per_iteration);
196 AccessWindowStatic sum_access(sum, 0, 0, 1, sum->dimension(1));
197 AccessWindowHorizontal output_access(output, 0, num_elems_processed_per_iteration);
198
199 bool window_changed = update_window_and_padding(win, input_access, sum_access, output_access);
200
201 output_access.set_valid_region(win, input->valid_region());
202
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000203 Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000204 return std::make_pair(err, win);
205}
206
207} // namespace
208
Chunosovd6afedc2017-11-06 22:09:45 +0700209/**< Grid size (obtained through auto-tuning) */
210const unsigned int CLLogits1DMaxShiftExpSumKernel::_grid_size = 64;
211/**< Vector size in the serial case (obtained through auto-tuning) */
212const unsigned int CLLogits1DMaxShiftExpSumKernel::_serial_vector_size = 8;
213/**< Vector size in the parallel case (obtained through auto-tuning, enables the best memory access pattern for Bifrost) .*/
214const unsigned int CLLogits1DMaxShiftExpSumKernel::_parallel_vector_size = 4;
215
216CLLogits1DMaxShiftExpSumKernel::CLLogits1DMaxShiftExpSumKernel()
217 : _input(nullptr), _max(nullptr), _output(nullptr), _sum(nullptr)
218{
219}
220
Sang-Hoon Park62eeb532019-10-29 13:13:19 +0000221void CLLogits1DMaxShiftExpSumKernel::configure(const ICLTensor *input, ICLTensor *max, ICLTensor *output, ICLTensor *sum, const SoftmaxKernelInfo &info)
Chunosovd6afedc2017-11-06 22:09:45 +0700222{
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000223 ARM_COMPUTE_ERROR_ON_NULLPTR(input, max, sum, output);
Chunosovd6afedc2017-11-06 22:09:45 +0700224
225 // Output auto initialization if not yet initialized
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000226 auto_init_if_empty(*sum->info(), input->info()->clone()->set_tensor_shape(max->info()->tensor_shape()));
227 auto_init_if_empty(*output->info(), *input->info()->clone());
Chunosovd6afedc2017-11-06 22:09:45 +0700228
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000229 // Perform validation step
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000230 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments_1DMaxShiftExpSum(input->info(), max->info(), output->info(), sum->info()));
Chunosovd6afedc2017-11-06 22:09:45 +0700231
232 _input = input;
233 _max = max;
234 _output = output;
235 _sum = sum;
236
Georgios Pinitas4c5469b2019-05-21 13:32:43 +0100237 const DataType dt = input->info()->data_type();
238 const UniformQuantizationInfo qinfo = input->info()->quantization_info().uniform();
239 const size_t reduction_dim_size = input->info()->dimension(0);
Sang-Hoon Park62eeb532019-10-29 13:13:19 +0000240 const float beta = info.beta;
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000241 const auto is_signed_qasymm8 = is_data_type_quantized_asymmetric_signed(info.input_data_type);
242 const int min_value = is_signed_qasymm8 ? CL_SCHAR_MIN : 0;
Chunosovd6afedc2017-11-06 22:09:45 +0700243
244 // Set build options
245 CLBuildOptions build_opts;
246 build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(dt));
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000247 build_opts.add_option("-DMIN_VALUE=" + support::cpp11::to_string(min_value));
248 build_opts.add_option_if(is_signed_qasymm8, "-DQASYMM8_SIGNED");
Chunosovd6afedc2017-11-06 22:09:45 +0700249 build_opts.add_option_if(dt == DataType::F16, "-DUSE_F16");
Chunosovd6afedc2017-11-06 22:09:45 +0700250 build_opts.add_option_if(is_data_type_float(dt) && (beta != 1.0f), "-DBETA=" + float_to_string_with_full_precision(beta));
Georgios Pinitas4c5469b2019-05-21 13:32:43 +0100251 build_opts.add_options_if(is_data_type_quantized_asymmetric(dt), prepare_quantized_softmax_build_options(qinfo.scale, beta).options());
Sang-Hoon Park62eeb532019-10-29 13:13:19 +0000252 build_opts.add_option_if(info.is_log, "-DLOG_SOFTMAX");
Chunosovd6afedc2017-11-06 22:09:45 +0700253
Anthony Barbierb6eb3532018-08-08 13:20:04 +0100254 cl::NDRange lws_hint(cl::NullRange);
Giorgio Arena4402cb92018-02-15 13:37:40 +0000255 std::string kernel_name = is_data_type_quantized_asymmetric(dt) ? std::string("softmax_layer_max_shift_exp_sum_quantized_serial") :
256 std::string("softmax_layer_max_shift_exp_sum_serial");
Chunosovd6afedc2017-11-06 22:09:45 +0700257 ParallelReductionInfo parallel_reduction_info = is_parallel_reduction(reduction_dim_size);
258 unsigned int vector_size = std::get<1>(parallel_reduction_info);
259
260 build_opts.add_option("-DVECTOR_SIZE=" + support::cpp11::to_string(vector_size));
261 build_opts.add_option("-DLOG_VECTOR_SIZE=" + support::cpp11::to_string(lround(log2(vector_size))));
262 build_opts.add_option_if((reduction_dim_size % vector_size) != 0, "-DNON_MULTIPLE_OF_VECTOR_SIZE");
263
264 // Configure parallel kernel if needed
265 if(std::get<0>(parallel_reduction_info))
266 {
Giorgio Arena4402cb92018-02-15 13:37:40 +0000267 kernel_name = is_data_type_quantized_asymmetric(dt) ? std::string("softmax_layer_max_shift_exp_sum_quantized_parallel") : std::string("softmax_layer_max_shift_exp_sum_parallel");
Chunosovd6afedc2017-11-06 22:09:45 +0700268 bool is_grid_size_pow2 = (_grid_size != 0) && ((_grid_size & (_grid_size - 1)) == 0);
269 build_opts.add_option_if(is_grid_size_pow2 && _grid_size <= 256, "-DGRID_SIZE=" + support::cpp11::to_string(_grid_size));
270
271 // Handle boundary conditions.
272 const unsigned int multiple_grid_size = (reduction_dim_size / vector_size) % _grid_size;
273 build_opts.add_option_if((multiple_grid_size != 0) || ((reduction_dim_size % vector_size) != 0), "-DNON_MULTIPLE_OF_GRID_SIZE");
Georgios Pinitas11f09992017-11-27 11:18:34 +0000274 // Setting _lws_hint in this way can also communicate grid_size to CLLogits1DMaxShiftExpSumKernel::run().
275 // A single workgroup performs reduction in dimension 0 in the parallel case, hence lws[0]==gws[0].
Anthony Barbierb6eb3532018-08-08 13:20:04 +0100276 lws_hint = cl::NDRange(_grid_size);
Chunosovd6afedc2017-11-06 22:09:45 +0700277 }
278
279 // Create kernel.
280 _kernel = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel(kernel_name, build_opts.options()));
281
282 // Set static arguments. Both the kernels use the same arguments
283 unsigned int idx = 4 * num_arguments_per_3D_tensor(); //Skip the input and output parameters
284 _kernel.setArg<cl_uint>(idx++, reduction_dim_size);
285
286 // Configure window
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000287 auto win_config = validate_and_configure_window_1DMaxShiftExpSum(input->info(), max->info(), output->info(), sum->info());
288 ARM_COMPUTE_ERROR_THROW_ON(win_config.first);
Anthony Barbierb6eb3532018-08-08 13:20:04 +0100289 ICLKernel::configure_internal(win_config.second, lws_hint);
Chunosovd6afedc2017-11-06 22:09:45 +0700290}
291
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000292Status CLLogits1DMaxShiftExpSumKernel::validate(const ITensorInfo *input, const ITensorInfo *max, const ITensorInfo *output, const ITensorInfo *sum)
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000293{
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000294 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments_1DMaxShiftExpSum(input, max, output, sum));
295 ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window_1DMaxShiftExpSum(input->clone().get(), max->clone().get(), output->clone().get(), sum->clone().get()).first);
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000296
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000297 return Status{};
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000298}
299
Chunosovd6afedc2017-11-06 22:09:45 +0700300CLLogits1DMaxShiftExpSumKernel::ParallelReductionInfo CLLogits1DMaxShiftExpSumKernel::is_parallel_reduction(size_t size)
301{
302 bool is_parallel_reduction = (size >= (_grid_size * _serial_vector_size)) && (_grid_size > 1);
303 unsigned int vector_size = is_parallel_reduction ? _parallel_vector_size : _serial_vector_size;
304 return std::make_tuple(is_parallel_reduction, vector_size);
305}
306
307void CLLogits1DMaxShiftExpSumKernel::run(const Window &window, cl::CommandQueue &queue)
308{
309 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
310 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(IKernel::window(), window);
311
312 // Collapse window in Z dimension
313 Window window_collapsed = window.collapse_if_possible(ICLKernel::window(), Window::DimZ);
314
315 // Reconfigure window in case of parallel reduction
316 ParallelReductionInfo parallel_reduction_info = is_parallel_reduction(_input->info()->dimension(0));
317 if(std::get<0>(parallel_reduction_info))
318 {
319 // To launch grid_size parallel workitems, steps.x should be modified as follows.
320 const unsigned int step = std::get<1>(parallel_reduction_info);
321 window_collapsed.set(Window::DimX, Window::Dimension(0, _grid_size * step, step));
322 }
323
324 // Get slices
325 Window slice = window_collapsed.first_slice_window_3D();
326 do
327 {
328 unsigned int idx = 0;
329 // Set inputs
330 add_3D_tensor_argument(idx, _input, slice);
331 add_3D_tensor_argument(idx, _max, slice);
332 add_3D_tensor_argument(idx, _output, slice);
333 add_3D_tensor_argument(idx, _sum, slice);
Anthony Barbierb6eb3532018-08-08 13:20:04 +0100334 enqueue(queue, *this, slice, lws_hint());
Chunosovd6afedc2017-11-06 22:09:45 +0700335 }
336 while(window_collapsed.slide_window_slice_3D(slice));
337}
338
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100339CLLogits1DNormKernel::CLLogits1DNormKernel()
340 : _input(nullptr), _sum(nullptr), _output(nullptr)
341{
342}
343
Sang-Hoon Park62eeb532019-10-29 13:13:19 +0000344void CLLogits1DNormKernel::configure(const ICLTensor *input, const ICLTensor *sum, ICLTensor *output, const SoftmaxKernelInfo &info)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100345{
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000346 ARM_COMPUTE_ERROR_ON_NULLPTR(input, sum, output);
Chunosovf450caa2017-11-08 16:09:35 +0700347
348 // Note: output should always have a scale of 1/256 and offset 0
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000349 const bool is_quantized_asymmetric = is_data_type_quantized_asymmetric(info.input_data_type);
350 const DataType output_data_type = info.input_data_type;
351 const QuantizationInfo allowed_quantization_info = get_softmax_output_quantization_info(info.input_data_type, info.is_log);
Georgios Pinitas4c5469b2019-05-21 13:32:43 +0100352 const UniformQuantizationInfo qinfo = input->info()->quantization_info().uniform();
Georgios Pinitasd368df32017-07-04 11:06:15 +0100353
354 // Output auto initialization if not yet initialized
Chunosovf450caa2017-11-08 16:09:35 +0700355 auto_init_if_empty(*output->info(),
Georgios Pinitas283c1792017-11-10 18:14:06 +0000356 input->info()->clone()->set_data_type(output_data_type).set_quantization_info(allowed_quantization_info));
Georgios Pinitasd368df32017-07-04 11:06:15 +0100357
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000358 // Perform validation step
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000359 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments_1DNorm(input->info(), sum->info(), output->info(), info));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100360
361 _input = input;
362 _sum = sum;
363 _output = output;
364
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000365 const auto is_signed_qasymm8 = is_data_type_quantized_asymmetric_signed(info.input_data_type);
366 const int min_value = is_signed_qasymm8 ? CL_SCHAR_MIN : 0;
367
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100368 // Set build options
Chunosovf450caa2017-11-08 16:09:35 +0700369 CLBuildOptions build_opts;
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000370 build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(info.input_data_type));
371 build_opts.add_option("-DMIN_VALUE=" + support::cpp11::to_string(min_value));
372 build_opts.add_option_if(is_data_type_quantized_asymmetric_signed(info.input_data_type), "-DQASYMM8_SIGNED");
Chunosovf450caa2017-11-08 16:09:35 +0700373 build_opts.add_options_if(is_quantized_asymmetric,
Sang-Hoon Park62eeb532019-10-29 13:13:19 +0000374 prepare_quantized_softmax_build_options(qinfo.scale, info.beta).options());
375 build_opts.add_option_if(info.is_log, "-DLOG_SOFTMAX");
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100376
377 // Create kernel
Chunosovf450caa2017-11-08 16:09:35 +0700378 std::string kernel_name = is_quantized_asymmetric ? "softmax_layer_norm_quantized" : "softmax_layer_norm";
379 _kernel = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel(kernel_name, build_opts.options()));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100380
381 // Configure window
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000382 auto win_config = validate_and_configure_window_1DNorm(input->info(), output->info(), sum->info(), info);
Giorgio Arenab8ab9972017-11-29 15:09:39 +0000383 ARM_COMPUTE_ERROR_THROW_ON(win_config.first);
Anthony Barbierb6eb3532018-08-08 13:20:04 +0100384 ICLKernel::configure_internal(win_config.second);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100385}
386
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000387Status CLLogits1DNormKernel::validate(const ITensorInfo *input, const ITensorInfo *sum, const ITensorInfo *output, const SoftmaxKernelInfo &info)
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000388{
Sang-Hoon Park0779fec2019-11-13 17:08:12 +0000389 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments_1DNorm(input, sum, output, info));
390 ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window_1DNorm(input->clone().get(), output->clone().get(), sum->clone().get(), info).first);
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000391
Georgios Pinitas631c41a2017-12-06 11:53:03 +0000392 return Status{};
Georgios Pinitas30902ed2017-11-14 15:32:57 +0000393}
394
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100395void CLLogits1DNormKernel::run(const Window &window, cl::CommandQueue &queue)
396{
397 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
398 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(IKernel::window(), window);
399
steniu010d523cc2017-07-13 14:24:23 +0100400 Window window_collapsed = window.collapse_if_possible(ICLKernel::window(), Window::DimZ);
401 Window slice = window_collapsed.first_slice_window_3D();
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100402
403 do
404 {
405 Window sum_slice = slice;
406 sum_slice.set(Window::DimX, Window::Dimension(0, 1, 1));
407
408 unsigned int idx = 0;
409 // Set inputs
steniu010d523cc2017-07-13 14:24:23 +0100410 add_3D_tensor_argument(idx, _input, slice);
411 add_3D_tensor_argument(idx, _sum, sum_slice);
412 add_3D_tensor_argument(idx, _output, slice);
Anthony Barbierb6eb3532018-08-08 13:20:04 +0100413 enqueue(queue, *this, slice, lws_hint());
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100414 }
steniu010d523cc2017-07-13 14:24:23 +0100415 while(window_collapsed.slide_window_slice_3D(slice));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100416}