blob: 990e4b67bc712e1be1f60d1da28c7e198e956cfb [file] [log] [blame]
Michele Di Giorgio4e09b382017-07-05 18:20:02 +01001/*
Michele Di Giorgiod9eaf612020-07-08 11:12:57 +01002 * Copyright (c) 2017-2020 Arm Limited.
Michele Di Giorgio4e09b382017-07-05 18:20:02 +01003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "arm_compute/core/NEON/kernels/NEQuantizationLayerKernel.h"
25
26#include "arm_compute/core/Error.h"
27#include "arm_compute/core/Helpers.h"
Michele Di Giorgio4e09b382017-07-05 18:20:02 +010028#include "arm_compute/core/Utils.h"
29#include "arm_compute/core/Validate.h"
30#include "arm_compute/core/Window.h"
Georgios Pinitasddb93bb2020-10-02 16:38:59 +010031#include "src/core/NEON/NEAsymm.h"
32#include "src/core/NEON/NEMath.h"
33#include "src/core/NEON/wrapper/wrapper.h"
Sang-Hoon Park68dd25f2020-10-19 16:00:11 +010034#include "src/core/helpers/AutoConfiguration.h"
35#include "src/core/helpers/WindowHelpers.h"
Michele Di Giorgio4e09b382017-07-05 18:20:02 +010036
Sang-Hoon Park68dd25f2020-10-19 16:00:11 +010037#include "src/core/CPP/Validate.h"
John Kesapidesadfb2732019-03-04 16:29:22 +000038
Michele Di Giorgio4e09b382017-07-05 18:20:02 +010039#include <arm_neon.h>
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +010040#include <map>
Michele Di Giorgio4e09b382017-07-05 18:20:02 +010041
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +010042namespace arm_compute
43{
Alex Gilday60954c62018-03-05 16:22:48 +000044namespace
45{
Sang-Hoon Parkfb6aaeb2019-11-27 15:26:44 +000046constexpr auto window_step = 16;
47
John Kesapidesadfb2732019-03-04 16:29:22 +000048Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output)
Alex Gilday60954c62018-03-05 16:22:48 +000049{
John Kesapidesadfb2732019-03-04 16:29:22 +000050 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output);
51 ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input);
Manuel Bottini4370cff2020-02-07 16:31:59 +000052 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::F16, DataType::F32);
John Kesapidesadfb2732019-03-04 16:29:22 +000053 ARM_COMPUTE_RETURN_ERROR_ON(output->tensor_shape().total_size() == 0);
Sang-Hoon Parkfb6aaeb2019-11-27 15:26:44 +000054 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::QASYMM16);
John Kesapidesadfb2732019-03-04 16:29:22 +000055 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input, output);
Alex Gilday60954c62018-03-05 16:22:48 +000056
57 return Status{};
58}
59
Manuel Bottini4370cff2020-02-07 16:31:59 +000060template <typename T>
61inline float32x4x4_t load_value(const T *input_ptr)
62{
63 using Tx16_t = typename wrapper::traits::neon_vector<T, 16>::type;
64 return arm_compute::convert_to_float32x4x4<Tx16_t>(wrapper::vloadq(input_ptr));
65}
66
67template <>
Michalis Spyroua4f378d2019-04-26 14:54:54 +010068inline float32x4x4_t load_value(const float *input_ptr)
Alex Gilday60954c62018-03-05 16:22:48 +000069{
John Kesapidesadfb2732019-03-04 16:29:22 +000070 return { wrapper::vloadq(input_ptr),
71 wrapper::vloadq(input_ptr + 4),
72 wrapper::vloadq(input_ptr + 8),
73 wrapper::vloadq(input_ptr + 12) };
Alex Gilday60954c62018-03-05 16:22:48 +000074}
John Kesapidesadfb2732019-03-04 16:29:22 +000075#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
Manuel Bottini4370cff2020-02-07 16:31:59 +000076template <>
77inline float32x4x4_t load_value(const float16_t *input_ptr)
John Kesapidesadfb2732019-03-04 16:29:22 +000078{
79 return { vcvt_f32_f16(wrapper::vload(input_ptr)),
80 vcvt_f32_f16(wrapper::vload(input_ptr + 4)),
81 vcvt_f32_f16(wrapper::vload(input_ptr + 8)),
82 vcvt_f32_f16(wrapper::vload(input_ptr + 12)) };
83}
84
85#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
Sang-Hoon Parkfb6aaeb2019-11-27 15:26:44 +000086
87template <typename element_type>
88using vector_type = wrapper::traits::neon_vector_t<element_type, window_step>;
89
90template <typename quantized_type>
91vector_type<quantized_type> vquantize_qasymm8(const float32x4x4_t &qv, const UniformQuantizationInfo &qi);
92
93template <>
94vector_type<uint8_t> vquantize_qasymm8<uint8_t>(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
95{
96 return vquantize(qv, qi);
97}
98
99template <>
100vector_type<int8_t> vquantize_qasymm8<int8_t>(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
101{
102 return vquantize_signed(qv, qi);
103}
104
Alex Gilday60954c62018-03-05 16:22:48 +0000105} // namespace
106
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100107NEQuantizationLayerKernel::NEQuantizationLayerKernel()
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100108 : _input(nullptr), _output(nullptr), _func(nullptr)
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100109{
110}
111
John Kesapidesadfb2732019-03-04 16:29:22 +0000112void NEQuantizationLayerKernel::configure(const ITensor *input, ITensor *output)
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100113{
John Kesapidesadfb2732019-03-04 16:29:22 +0000114 ARM_COMPUTE_ERROR_ON_NULLPTR(input, output);
115 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), output->info()));
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100116
John Kesapidesadfb2732019-03-04 16:29:22 +0000117 _input = input;
118 _output = output;
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100119
Manuel Bottini4370cff2020-02-07 16:31:59 +0000120 static const std::map<std::string, QuantizationFunctionExecutorPtr> quant_map =
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100121 {
Manuel Bottini4370cff2020-02-07 16:31:59 +0000122 { "op_QASYMM8_QASYMM8", &NEQuantizationLayerKernel::run_quantize_qasymm8<uint8_t, uint8_t> },
123 { "op_QASYMM8_QASYMM8_SIGNED", &NEQuantizationLayerKernel::run_quantize_qasymm8<uint8_t, int8_t> },
124 { "op_QASYMM8_QASYMM16", &NEQuantizationLayerKernel::run_quantize_qasymm16<uint8_t> },
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100125
Manuel Bottini4370cff2020-02-07 16:31:59 +0000126 { "op_QASYMM8_SIGNED_QASYMM8", &NEQuantizationLayerKernel::run_quantize_qasymm8<int8_t, uint8_t> },
127 { "op_QASYMM8_SIGNED_QASYMM8_SIGNED", &NEQuantizationLayerKernel::run_quantize_qasymm8<int8_t, int8_t> },
128 { "op_QASYMM8_SIGNED_QASYMM16", &NEQuantizationLayerKernel::run_quantize_qasymm16<int8_t> },
129
130 { "op_F32_QASYMM8", &NEQuantizationLayerKernel::run_quantize_qasymm8<float, uint8_t> },
131 { "op_F32_QASYMM8_SIGNED", &NEQuantizationLayerKernel::run_quantize_qasymm8<float, int8_t> },
132 { "op_F32_QASYMM16", &NEQuantizationLayerKernel::run_quantize_qasymm16<float> },
133
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100134#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
Manuel Bottini4370cff2020-02-07 16:31:59 +0000135 { "op_F16_QASYMM8", &NEQuantizationLayerKernel::run_quantize_qasymm8<float16_t, uint8_t> },
136 { "op_F16_QASYMM8_SIGNED", &NEQuantizationLayerKernel::run_quantize_qasymm8<float16_t, int8_t> },
137 { "op_F16_QASYMM16", &NEQuantizationLayerKernel::run_quantize_qasymm16<float16_t> },
138#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC*/
139 };
140
141 std::string function_to_call("op_");
142 function_to_call += string_from_data_type(_input->info()->data_type()) + "_";
143 function_to_call += string_from_data_type(_output->info()->data_type());
144
145 auto it = quant_map.find(function_to_call);
146
147 if(it == quant_map.end())
148 {
149 ARM_COMPUTE_ERROR("Unsupported combination of input and output data types");
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100150 }
Manuel Bottini4370cff2020-02-07 16:31:59 +0000151 _func = it->second;
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100152
Alex Gilday60954c62018-03-05 16:22:48 +0000153 // Configure kernel window
John Kesapidesadfb2732019-03-04 16:29:22 +0000154 Window win_config = calculate_max_window(*input->info(), Steps());
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100155
John Kesapidesadfb2732019-03-04 16:29:22 +0000156 Coordinates coord;
157 coord.set_num_dimensions(output->info()->num_dimensions());
158 output->info()->set_valid_region(ValidRegion(coord, output->info()->tensor_shape()));
Gian Marco Iodice06b184a2017-08-29 16:05:25 +0100159
John Kesapidesadfb2732019-03-04 16:29:22 +0000160 INEKernel::configure(win_config);
Alex Gilday60954c62018-03-05 16:22:48 +0000161}
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100162
John Kesapidesadfb2732019-03-04 16:29:22 +0000163Status NEQuantizationLayerKernel::validate(const ITensorInfo *input, const ITensorInfo *output)
Alex Gilday60954c62018-03-05 16:22:48 +0000164{
John Kesapidesadfb2732019-03-04 16:29:22 +0000165 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, output));
Alex Gilday60954c62018-03-05 16:22:48 +0000166 return Status{};
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100167}
168
Sang-Hoon Parkfb6aaeb2019-11-27 15:26:44 +0000169template <typename TIn, typename TOut>
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100170void NEQuantizationLayerKernel::run_quantize_qasymm8(const Window &window)
John Kesapidesadfb2732019-03-04 16:29:22 +0000171{
Sang-Hoon Parkfb6aaeb2019-11-27 15:26:44 +0000172 const auto window_start_x = static_cast<int>(window.x().start());
173 const auto window_end_x = static_cast<int>(window.x().end());
John Kesapidesadfb2732019-03-04 16:29:22 +0000174
Manuel Bottini4370cff2020-02-07 16:31:59 +0000175 const UniformQuantizationInfo uqinfo_in = _input->info()->quantization_info().uniform();
176 UniformQuantizationInfo uqinfo = _output->info()->quantization_info().uniform();
177 if(is_data_type_quantized_asymmetric(_input->info()->data_type()))
178 {
179 uqinfo = compute_requantization_scale_offset(uqinfo_in, uqinfo);
180 }
John Kesapidesadfb2732019-03-04 16:29:22 +0000181#ifdef __aarch64__
182 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_NEAREST_EVEN;
183#else //__aarch64__
184 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_ZERO;
185#endif //__aarch64__
186
187 // Collapse window and reset first dimension to handle tail calculations manually
188 Window win_collapsed = window.collapse_if_possible(window, Window::DimZ);
189 win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1));
190
191 Iterator input(_input, win_collapsed);
192 Iterator output(_output, win_collapsed);
Michalis Spyroua4f378d2019-04-26 14:54:54 +0100193 execute_window_loop(win_collapsed, [&](const Coordinates &)
John Kesapidesadfb2732019-03-04 16:29:22 +0000194 {
Sang-Hoon Parkfb6aaeb2019-11-27 15:26:44 +0000195 auto input_ptr = reinterpret_cast<const TIn *>(input.ptr());
196 auto output_ptr = reinterpret_cast<TOut *>(output.ptr());
John Kesapidesadfb2732019-03-04 16:29:22 +0000197
198 int x = window_start_x;
199 for(; x <= (window_end_x - window_step); x += window_step)
200 {
Sang-Hoon Parkfb6aaeb2019-11-27 15:26:44 +0000201 wrapper::vstore(&output_ptr[x], vquantize_qasymm8<TOut>(load_value(&input_ptr[x]), uqinfo));
John Kesapidesadfb2732019-03-04 16:29:22 +0000202 }
203 // Compute left-over elements
204 for(; x < window_end_x; ++x)
205 {
Sang-Hoon Parkfb6aaeb2019-11-27 15:26:44 +0000206 output_ptr[x] = Qasymm8QuantizationHelper<TOut>::quantize(input_ptr[x], uqinfo, rounding_policy);
John Kesapidesadfb2732019-03-04 16:29:22 +0000207 }
208 },
209 input, output);
210}
211
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100212template <typename T>
213void NEQuantizationLayerKernel::run_quantize_qasymm16(const Window &window)
214{
Sang-Hoon Parkfb6aaeb2019-11-27 15:26:44 +0000215 const auto window_start_x = static_cast<int>(window.x().start());
216 const auto window_end_x = static_cast<int>(window.x().end());
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100217
Manuel Bottini4370cff2020-02-07 16:31:59 +0000218 const UniformQuantizationInfo uqinfo_in = _input->info()->quantization_info().uniform();
219 UniformQuantizationInfo uqinfo = _output->info()->quantization_info().uniform();
220 if(is_data_type_quantized_asymmetric(_input->info()->data_type()))
221 {
222 uqinfo = compute_requantization_scale_offset(uqinfo_in, uqinfo);
223 }
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100224#ifdef __aarch64__
225 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_NEAREST_EVEN;
226#else //__aarch64__
227 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_ZERO;
228#endif //__aarch64__
229
230 // Collapse window and reset first dimension to handle tail calculations manually
231 Window win_collapsed = window.collapse_if_possible(window, Window::DimZ);
232 win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1));
233
234 Iterator input(_input, win_collapsed);
235 Iterator output(_output, win_collapsed);
236 execute_window_loop(win_collapsed, [&](const Coordinates &)
237 {
238 auto input_ptr = reinterpret_cast<const T *>(input.ptr());
239 auto output_ptr = reinterpret_cast<uint16_t *>(output.ptr());
240
241 int x = window_start_x;
242 for(; x <= (window_end_x - window_step); x += window_step)
243 {
244 uint16x8x2_t tmp = vquantize_qasymm16(load_value(&input_ptr[x]), uqinfo);
245 vst1q_u16(&output_ptr[x], tmp.val[0]);
246 vst1q_u16(&output_ptr[x + 8], tmp.val[1]);
247 }
248 // Compute left-over elements
249 for(; x < window_end_x; ++x)
250 {
251 output_ptr[x] = quantize_qasymm16(input_ptr[x], uqinfo, rounding_policy);
252 }
253 },
254 input, output);
255}
256
Moritz Pflanzerc186b572017-09-07 09:48:04 +0100257void NEQuantizationLayerKernel::run(const Window &window, const ThreadInfo &info)
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100258{
Moritz Pflanzerc186b572017-09-07 09:48:04 +0100259 ARM_COMPUTE_UNUSED(info);
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100260 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
261 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window);
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100262 ARM_COMPUTE_ERROR_ON(_func == nullptr);
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100263
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100264 (this->*_func)(window);
Michele Di Giorgio4e09b382017-07-05 18:20:02 +0100265}
Michele Di Giorgiod64a46c2019-10-01 12:25:49 +0100266} // namespace arm_compute