COMPMID-2179 New generic depthwise convolution for NEON F32 NHWC
Change-Id: I2b883785c0500d4bdb6ee4700382ee058be2cd36
Signed-off-by: Giorgio Arena <giorgio.arena@arm.com>
Reviewed-on: https://review.mlplatform.org/c/1538
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
diff --git a/arm_compute/core/NEON/NEKernels.h b/arm_compute/core/NEON/NEKernels.h
index e41f299..8fddec2 100644
--- a/arm_compute/core/NEON/NEKernels.h
+++ b/arm_compute/core/NEON/NEKernels.h
@@ -53,6 +53,7 @@
#include "arm_compute/core/NEON/kernels/NEDepthConvertLayerKernel.h"
#include "arm_compute/core/NEON/kernels/NEDepthToSpaceLayerKernel.h"
#include "arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayer3x3Kernel.h"
+#include "arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.h"
#include "arm_compute/core/NEON/kernels/NEDepthwiseIm2ColKernel.h"
#include "arm_compute/core/NEON/kernels/NEDepthwiseVectorToTensorKernel.h"
#include "arm_compute/core/NEON/kernels/NEDepthwiseWeightsReshapeKernel.h"
diff --git a/arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.h b/arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.h
new file mode 100644
index 0000000..63635b3
--- /dev/null
+++ b/arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef __ARM_COMPUTE_NEDEPTHWISECONVOLUTIONKERNEL_H__
+#define __ARM_COMPUTE_NEDEPTHWISECONVOLUTIONKERNEL_H__
+
+#include "arm_compute/core/NEON/INEKernel.h"
+
+namespace arm_compute
+{
+// Forward declarations
+class ITensor;
+
+/** Interface for the kernel to run a depthwise convolution on a tensor. */
+class NEDepthwiseConvolutionLayerKernel : public INEKernel
+{
+public:
+ const char *name() const override
+ {
+ return "NEDepthwiseConvolutionLayerKernel";
+ }
+ /** Default constructor */
+ NEDepthwiseConvolutionLayerKernel();
+ /** Prevent instances of this class from being copied (As this class contains pointers) */
+ NEDepthwiseConvolutionLayerKernel(const NEDepthwiseConvolutionLayerKernel &) = delete;
+ /** Prevent instances of this class from being copied (As this class contains pointers) */
+ NEDepthwiseConvolutionLayerKernel &operator=(const NEDepthwiseConvolutionLayerKernel &) = delete;
+ /** Default Move Constructor. */
+ NEDepthwiseConvolutionLayerKernel(NEDepthwiseConvolutionLayerKernel &&) = default;
+ /** Default move assignment operator */
+ NEDepthwiseConvolutionLayerKernel &operator=(NEDepthwiseConvolutionLayerKernel &&) = default;
+ /** Initialize the function's source, destination and parameters.
+ *
+ * @note Supported data layouts: NHWC
+ *
+ * @param[in] input Source tensor. DataType supported: F32.
+ * @param[in] weights Weights tensor. This is a 3D tensor with dimensions [IFM, W, H]. Data type supported: Same as @p input.
+ * @param[in] biases Biases tensor. A 1D tensor with dimensions [IFM]. Must be nullptr if not needed. Data type supported: Same as @p input.
+ * @param[out] output Destination tensor. Data type supported: Same as @p input.
+ * @param[in] conv_info Padding and stride information to use for the convolution.
+ * @param[in] depth_multiplier (Optional) Multiplier to apply to the input's depth in order to retrieve the output's depth. Defaults to 1.
+ * @param[in] dilation (Optional) Dilation, in elements, across x and y. Defaults to (1, 1).
+ *
+ */
+ void configure(const ITensor *input, const ITensor *weights, const ITensor *biases, ITensor *output, const PadStrideInfo &conv_info, unsigned int depth_multiplier = 1,
+ const Size2D &dilation = Size2D(1U, 1U));
+ /** Static function to check if given info will lead to a valid configuration of @ref NEDepthwiseConvolutionLayerKernel
+ *
+ * @note Supported data layouts: NHWC
+ *
+ * @param[in] input Source tensor info. DataType supported: F32.
+ * @param[in] weights Weights tensor info. This is a 3D tensor with dimensions [IFM, W, H]. Data type supported: Same as @p input.
+ * @param[in] biases Biases tensor info. A 1D tensor with dimensions [IFM]. Must be nullptr if not needed. Data type supported: Same as @p input.
+ * @param[in] output Destination tensor info. Data type supported: Same as @p input.
+ * @param[in] conv_info Padding and stride information to use for the convolution.
+ * @param[in] depth_multiplier (Optional) Multiplier to apply to the input's depth in order to retrieve the output's depth. Defaults to 1.
+ * @param[in] dilation (Optional) Dilation, in elements, across x and y. Defaults to (1, 1).
+ *
+ * @return a status
+ */
+ static Status validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info, unsigned int depth_multiplier = 1,
+ const Size2D &dilation = Size2D(1U, 1U));
+
+ // Inherited methods overridden:
+ void run(const Window &window, const ThreadInfo &info) override;
+ BorderSize border_size() const override;
+
+private:
+ template <typename T, int S, bool has_biases>
+ void run_depthwise(const Window &window);
+
+ /** Common signature for all the specialised depthwise convolution functions
+ *
+ * @param[in] window Region on which to execute the kernel.
+ */
+ using DepthwiseFunctionPtr = void (NEDepthwiseConvolutionLayerKernel::*)(const Window &window);
+
+ DepthwiseFunctionPtr _func;
+ BorderSize _border_size;
+ const ITensor *_input;
+ const ITensor *_weights;
+ const ITensor *_biases;
+ ITensor *_output;
+ PadStrideInfo _conv_info;
+ unsigned int _depth_multiplier;
+ Size2D _dilation;
+};
+} // namespace arm_compute
+#endif /* __ARM_COMPUTE_NEDEPTHWISECONVOLUTIONKERNEL_H__ */
diff --git a/arm_compute/runtime/NEON/functions/NEDepthwiseConvolutionLayer.h b/arm_compute/runtime/NEON/functions/NEDepthwiseConvolutionLayer.h
index 715f4f5..5b0d1ba 100644
--- a/arm_compute/runtime/NEON/functions/NEDepthwiseConvolutionLayer.h
+++ b/arm_compute/runtime/NEON/functions/NEDepthwiseConvolutionLayer.h
@@ -25,6 +25,7 @@
#define __ARM_COMPUTE_NEDEPTHWISECONVOLUTION_H__
#include "arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayer3x3Kernel.h"
+#include "arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.h"
#include "arm_compute/core/NEON/kernels/NEDepthwiseIm2ColKernel.h"
#include "arm_compute/core/NEON/kernels/NEDepthwiseVectorToTensorKernel.h"
#include "arm_compute/core/NEON/kernels/NEDepthwiseWeightsReshapeKernel.h"
@@ -280,6 +281,10 @@
/** Basic function to execute a generic depthwise convolution. This function calls the following NEON kernels:
*
+ * If data type is F32 and data layout is NHWC:
+ * -# @ref NEDepthwiseConvolutionLayerKernel
+ *
+ * Otherwise:
* -# @ref NEDepthwiseIm2ColKernel
* -# @ref NEDepthwiseWeightsReshapeKernel
* -# @ref NEGEMMMatrixVectorMultiplyKernel
@@ -339,8 +344,10 @@
NEDepthwiseIm2ColKernel _im2col_kernel;
NEDepthwiseWeightsReshapeKernel _weights_reshape_kernel;
NEGEMMMatrixVectorMultiplyKernel _v2mm_kernel;
+ NEDepthwiseConvolutionLayerKernel _depthwise_conv_kernel;
NEDepthwiseVectorToTensorKernel _vector_to_tensor_kernel;
NEDirectConvolutionLayerOutputStageKernel _output_stage_kernel;
+ NEFillBorderKernel _fill_border;
NEFillBorderKernel _v2mm_input_fill_border;
NEFillBorderKernel _v2mm_weights_fill_border;
NEPermute _permute_input;
@@ -358,6 +365,7 @@
bool _is_quantized;
bool _is_nhwc;
bool _is_activationlayer_enabled;
+ bool _is_optimized;
const ITensor *_original_weights;
};
} // namespace arm_compute
diff --git a/src/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.cpp b/src/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.cpp
new file mode 100644
index 0000000..feb2071
--- /dev/null
+++ b/src/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.cpp
@@ -0,0 +1,330 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.h"
+
+#include "arm_compute/core/AccessWindowStatic.h"
+#include "arm_compute/core/NEON/wrapper/traits.h"
+#include "arm_compute/core/NEON/wrapper/wrapper.h"
+#include "arm_compute/core/utils/misc/ShapeCalculator.h"
+
+namespace arm_compute
+{
+namespace
+{
+template <typename T, int S, bool has_biases>
+void depthwise_loop_multiplier1(const ITensor *input, const ITensor *weights, const ITensor *biases, ITensor *output, const PadStrideInfo &conv_info,
+ const Size2D &dilation, const Window &window)
+{
+ using VectorType = typename wrapper::traits::neon_vector<T, S>::type;
+ using TagType = typename wrapper::traits::neon_vector<T, S>::tag_type;
+
+ const size_t input_stride_y = input->info()->strides_in_bytes().y();
+ const size_t input_stride_z = input->info()->strides_in_bytes().z();
+ const size_t input_max_offset = input->info()->strides_in_bytes().z() * input->info()->dimension(2) - (input->info()->padding().bottom + input->info()->padding().top) *
+ input->info()->strides_in_bytes().y();
+ const size_t weights_width = weights->info()->dimension(1);
+ const size_t weights_height = weights->info()->dimension(2);
+ const size_t weights_stride_y = weights->info()->strides_in_bytes().y();
+ const size_t weights_stride_z = weights->info()->strides_in_bytes().z();
+ const size_t conv_stride_x = conv_info.stride().first;
+ const size_t conv_stride_y = conv_info.stride().second;
+ const size_t conv_pad_left = conv_info.pad_left();
+ const size_t conv_pad_top = conv_info.pad_top();
+
+ Window win_input = window;
+ win_input.set(Window::DimY, Window::Dimension(0, 0, 0));
+ win_input.set(Window::DimZ, Window::Dimension(0, 0, 0));
+
+ Window win_weights = win_input;
+ win_weights.set(3, Window::Dimension(0, 0, 0));
+
+ Iterator input_it(input, win_input);
+ Iterator weights_it(weights, win_weights);
+ Iterator output_it(output, window);
+ Iterator biases_it{};
+
+ if(has_biases)
+ {
+ biases_it = Iterator(biases, win_weights);
+ }
+
+ execute_window_loop(window, [&](const Coordinates & id)
+ {
+ VectorType acc = wrapper::vdup_n(static_cast<T>(0), TagType{});
+
+ const int input_y = id.y() * conv_stride_x - conv_pad_left;
+ const int input_z = id.z() * conv_stride_y - conv_pad_top;
+ int input_offset = input_y * input_stride_y + input_z * input_stride_z;
+
+ auto weights_ptr = weights_it.ptr();
+ for(size_t h = 0; h < weights_height; ++h)
+ {
+ int offs = input_offset;
+ for(size_t w = 0; w < weights_width; ++w)
+ {
+ const auto input_vals = wrapper::vload(reinterpret_cast<T *>(input_it.ptr() + std::min(static_cast<size_t>(offs), input_max_offset)));
+ const auto weights_vals = wrapper::vload(reinterpret_cast<T *>(weights_ptr + w * weights_stride_y));
+
+ acc = wrapper::vmla(acc, weights_vals, input_vals);
+ offs += dilation.x() * input_stride_y;
+ }
+
+ weights_ptr += weights_stride_z;
+ input_offset += dilation.y() * input_stride_z;
+ }
+
+ if(has_biases)
+ {
+ const auto biases_vals = wrapper::vload(reinterpret_cast<T *>(biases_it.ptr()));
+ acc = wrapper::vadd(acc, biases_vals);
+ }
+
+ wrapper::vstore(reinterpret_cast<T *>(output_it.ptr()), acc);
+ },
+ input_it, weights_it, biases_it, output_it);
+}
+
+template <typename T, bool has_biases>
+void depthwise_loop_generic(const ITensor *input, const ITensor *weights, const ITensor *biases, ITensor *output, const PadStrideInfo &conv_info,
+ const Size2D &dilation, unsigned int depth_multiplier, const Window &window)
+{
+ const size_t input_stride_y = input->info()->strides_in_bytes().y();
+ const size_t input_stride_z = input->info()->strides_in_bytes().z();
+ const size_t input_max_offset = input->info()->strides_in_bytes().z() * input->info()->dimension(2) - (input->info()->padding().bottom + input->info()->padding().top) *
+ input->info()->strides_in_bytes().y();
+ const size_t weights_width = weights->info()->dimension(1);
+ const size_t weights_height = weights->info()->dimension(2);
+ const size_t weights_stride_y = weights->info()->strides_in_bytes().y();
+ const size_t weights_stride_z = weights->info()->strides_in_bytes().z();
+ const size_t conv_stride_x = conv_info.stride().first;
+ const size_t conv_stride_y = conv_info.stride().second;
+ const size_t conv_pad_left = conv_info.pad_left();
+ const size_t conv_pad_top = conv_info.pad_top();
+
+ Window win_input = window;
+ win_input.set(Window::DimY, Window::Dimension(0, 0, 0));
+ win_input.set(Window::DimZ, Window::Dimension(0, 0, 0));
+
+ Window win_weights = win_input;
+ win_weights.set(3, Window::Dimension(0, 0, 0));
+
+ win_input.set_dimension_step(Window::DimX, 1);
+
+ Iterator input_it(input, win_input);
+ Iterator weights_it(weights, win_weights);
+ Iterator output_it(output, window);
+ Iterator biases_it{};
+
+ if(has_biases)
+ {
+ biases_it = Iterator(biases, win_weights);
+ }
+
+ execute_window_loop(window, [&](const Coordinates & id)
+ {
+ std::vector<T> acc(depth_multiplier, static_cast<T>(0));
+
+ const int input_y = id.y() * conv_stride_x - conv_pad_left;
+ const int input_z = id.z() * conv_stride_y - conv_pad_top;
+ int input_offset = input_y * input_stride_y + input_z * input_stride_z;
+
+ auto weights_ptr = weights_it.ptr();
+ for(size_t h = 0; h < weights_height; ++h)
+ {
+ int offs = input_offset;
+ for(size_t w = 0; w < weights_width; ++w)
+ {
+ const auto input_val = *(reinterpret_cast<T *>(input_it.ptr() + std::min(static_cast<size_t>(offs), input_max_offset)));
+
+ for(size_t m = 0; m < depth_multiplier; ++m)
+ {
+ const auto weights_val = *(reinterpret_cast<T *>(weights_ptr + m * sizeof(T) + w * weights_stride_y));
+ acc.at(m) = std::fma(weights_val, input_val, acc.at(m));
+ }
+
+ offs += dilation.x() * input_stride_y;
+ }
+
+ weights_ptr += weights_stride_z;
+ input_offset += dilation.y() * input_stride_z;
+ }
+
+ if(has_biases)
+ {
+ for(size_t m = 0; m < depth_multiplier; ++m)
+ {
+ const auto biases_val = *(reinterpret_cast<T *>(biases_it.ptr() + m * sizeof(T)));
+ *(reinterpret_cast<T *>(output_it.ptr() + m * sizeof(T))) = acc.at(m) + biases_val;
+ }
+ }
+ else
+ {
+ for(size_t m = 0; m < depth_multiplier; ++m)
+ {
+ *(reinterpret_cast<T *>(output_it.ptr() + m * sizeof(T))) = acc.at(m);
+ }
+ }
+ },
+ input_it, weights_it, biases_it, output_it);
+}
+
+Status validate_arguments(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info, unsigned int depth_multiplier,
+ const Size2D &dilation)
+{
+ ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output);
+ ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights, output);
+ ARM_COMPUTE_RETURN_ERROR_ON(depth_multiplier == 0);
+ ARM_COMPUTE_RETURN_ERROR_ON((input->dimension(0) * depth_multiplier) != weights->dimension(0));
+ ARM_COMPUTE_RETURN_ERROR_ON((dilation.x() < 1) || (dilation.y() < 1));
+ ARM_COMPUTE_RETURN_ERROR_ON((conv_info.stride().first < 1) || (conv_info.stride().second < 1));
+
+ if(biases != nullptr)
+ {
+ ARM_COMPUTE_RETURN_ERROR_ON(biases->num_dimensions() > 1);
+ ARM_COMPUTE_RETURN_ERROR_ON(biases->dimension(0) != weights->dimension(0));
+ }
+
+ if(output->total_size() != 0)
+ {
+ const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input, *weights, conv_info, depth_multiplier, dilation);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(output->tensor_shape(), output_shape);
+ }
+
+ return Status{};
+}
+} // namespace
+
+std::pair<Status, Window> validate_and_configure_window(ITensorInfo *input, ITensorInfo *weights, ITensorInfo *biases,
+ ITensorInfo *output, const PadStrideInfo &conv_info,
+ unsigned int depth_multiplier, const Size2D &dilation)
+{
+ // Get convolved dimensions
+ const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input, *weights, conv_info, depth_multiplier, dilation);
+
+ // Output auto inizialitation if not yet initialized
+ auto_init_if_empty(*output, input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape));
+
+ // Configure kernel window (generic)
+ const unsigned int num_elems_read_per_iteration = (depth_multiplier == 1) ? 8 / element_size_from_data_type(input->data_type()) : 1;
+ const unsigned int num_elems_written_per_iteration = num_elems_read_per_iteration * depth_multiplier;
+
+ // Configure kernel window
+ Window win = calculate_max_window(*output, Steps(num_elems_written_per_iteration));
+
+ AccessWindowStatic input_access(input, 0, -conv_info.pad_left(), ceil_to_multiple(num_elems_read_per_iteration, input->dimension(0)),
+ input->dimension(1) + std::max(std::max(conv_info.pad_right(), conv_info.pad_bottom()), conv_info.pad_top()));
+ AccessWindowHorizontal weights_access(weights, 0, num_elems_written_per_iteration);
+ AccessWindowHorizontal output_access(output, 0, num_elems_written_per_iteration);
+
+ bool window_changed = update_window_and_padding(win, input_access, weights_access, output_access);
+
+ if(biases != nullptr)
+ {
+ AccessWindowHorizontal biases_access(biases, 0, num_elems_written_per_iteration);
+ window_changed |= update_window_and_padding(win, biases_access);
+ }
+
+ output_access.set_valid_region(win, ValidRegion(Coordinates(), output->tensor_shape()));
+
+ Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};
+ return std::make_pair(err, win);
+}
+
+NEDepthwiseConvolutionLayerKernel::NEDepthwiseConvolutionLayerKernel()
+ : _func(), _border_size(0), _input(), _weights(), _biases(), _output(), _conv_info(), _depth_multiplier(1), _dilation()
+{
+}
+
+BorderSize NEDepthwiseConvolutionLayerKernel::border_size() const
+{
+ return _border_size;
+}
+
+void NEDepthwiseConvolutionLayerKernel::configure(const ITensor *input, const ITensor *weights, const ITensor *biases, ITensor *output,
+ const PadStrideInfo &conv_info, unsigned int depth_multiplier, const Size2D &dilation)
+{
+ ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
+ ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), weights->info(), (biases != nullptr) ? biases->info() : nullptr, output->info(), conv_info, depth_multiplier, dilation));
+
+ _input = input;
+ _weights = weights;
+ _biases = biases;
+ _output = output;
+ _conv_info = conv_info;
+ _depth_multiplier = depth_multiplier;
+ _border_size = BorderSize(_conv_info.pad_left(), 0, std::max(std::max(conv_info.pad_right(), conv_info.pad_bottom()), conv_info.pad_top()), 0);
+ _dilation = dilation;
+
+ switch(_input->info()->data_type())
+ {
+ case DataType::F32:
+ _func = (biases != nullptr) ? &NEDepthwiseConvolutionLayerKernel::run_depthwise<float, 2, true> : &NEDepthwiseConvolutionLayerKernel::run_depthwise<float, 2, false>;
+ break;
+ default:
+ ARM_COMPUTE_ERROR("Data type not supported");
+ break;
+ }
+
+ auto win_config = validate_and_configure_window(_input->info(), _weights->info(), (biases != nullptr) ? biases->info() : nullptr, _output->info(), _conv_info, _depth_multiplier, dilation);
+ ARM_COMPUTE_ERROR_THROW_ON(win_config.first);
+ INEKernel::configure(win_config.second);
+}
+
+Status NEDepthwiseConvolutionLayerKernel::validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info,
+ unsigned int depth_multiplier,
+ const Size2D &dilation)
+{
+ ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, weights, biases, output, conv_info, depth_multiplier, dilation));
+ ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(input->clone().get(), weights->clone().get(), (biases != nullptr) ? biases->clone().get() : nullptr, output->clone().get(), conv_info,
+ depth_multiplier, dilation)
+ .first);
+ return Status{};
+}
+
+void NEDepthwiseConvolutionLayerKernel::run(const Window &window, const ThreadInfo &info)
+{
+ ARM_COMPUTE_UNUSED(info);
+ ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
+ ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window);
+
+ (this->*_func)(window);
+}
+
+template <typename T, int S, bool has_biases>
+void NEDepthwiseConvolutionLayerKernel::run_depthwise(const Window &window)
+{
+ ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
+ ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window);
+
+ if(_depth_multiplier == 1)
+ {
+ depthwise_loop_multiplier1<T, S, has_biases>(_input, _weights, _biases, _output, _conv_info, _dilation, window);
+ }
+ else
+ {
+ depthwise_loop_generic<T, has_biases>(_input, _weights, _biases, _output, _conv_info, _dilation, _depth_multiplier, window);
+ }
+}
+} // namespace arm_compute
diff --git a/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp b/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp
index 001bece..c2ed901 100644
--- a/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp
+++ b/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp
@@ -689,9 +689,10 @@
}
NEDepthwiseConvolutionLayer::NEDepthwiseConvolutionLayer()
- : _im2col_kernel(), _weights_reshape_kernel(), _v2mm_kernel(), _vector_to_tensor_kernel(), _output_stage_kernel(), _v2mm_input_fill_border(), _v2mm_weights_fill_border(), _permute_input(),
- _permute_weights(), _permute_output(), _activationlayer_function(), _input_reshaped(), _weights_reshaped(), _v2mm_output(), _output_reshaped(), _permuted_input(), _permuted_weights(),
- _permuted_output(), _is_prepared(false), _is_quantized(false), _is_nhwc(false), _is_activationlayer_enabled(false), _original_weights(nullptr)
+ : _im2col_kernel(), _weights_reshape_kernel(), _v2mm_kernel(), _depthwise_conv_kernel(), _vector_to_tensor_kernel(), _output_stage_kernel(), _fill_border(), _v2mm_input_fill_border(),
+ _v2mm_weights_fill_border(), _permute_input(), _permute_weights(), _permute_output(), _activationlayer_function(), _input_reshaped(), _weights_reshaped(), _v2mm_output(), _output_reshaped(),
+ _permuted_input(), _permuted_weights(), _permuted_output(), _is_prepared(false), _is_quantized(false), _is_nhwc(false), _is_activationlayer_enabled(false), _is_optimized(false),
+ _original_weights(nullptr)
{
}
@@ -703,124 +704,136 @@
ARM_COMPUTE_ERROR_THROW_ON(NEDepthwiseConvolutionLayer::validate(input->info(), weights->info(), (biases == nullptr) ? nullptr : biases->info(),
output->info(), conv_info, depth_multiplier, act_info, dilation));
- _is_nhwc = input->info()->data_layout() == DataLayout::NHWC;
+ _is_nhwc = input->info()->data_layout() == DataLayout::NHWC;
+ _is_optimized = _is_nhwc && input->info()->data_type() == DataType::F32;
- ITensor *input_to_use = input;
- const ITensor *weights_to_use = weights;
- ITensor *output_to_use = output;
-
- if(_is_nhwc)
+ if(!_is_optimized)
{
- _permute_input.configure(input, &_permuted_input, PermutationVector(1U, 2U, 0U));
- _permuted_input.info()->set_data_layout(DataLayout::NCHW);
- input_to_use = &_permuted_input;
+ ITensor *input_to_use = input;
+ const ITensor *weights_to_use = weights;
+ ITensor *output_to_use = output;
- _permute_weights.configure(weights, &_permuted_weights, PermutationVector(1U, 2U, 0U));
- _permuted_weights.info()->set_data_layout(DataLayout::NCHW);
- weights_to_use = &_permuted_weights;
+ if(_is_nhwc)
+ {
+ _permute_input.configure(input, &_permuted_input, PermutationVector(1U, 2U, 0U));
+ _permuted_input.info()->set_data_layout(DataLayout::NCHW);
+ input_to_use = &_permuted_input;
+
+ _permute_weights.configure(weights, &_permuted_weights, PermutationVector(1U, 2U, 0U));
+ _permuted_weights.info()->set_data_layout(DataLayout::NCHW);
+ weights_to_use = &_permuted_weights;
+ }
+
+ const size_t weights_w = weights_to_use->info()->dimension(0);
+ const size_t weights_h = weights_to_use->info()->dimension(1);
+ const size_t weights_z = weights_to_use->info()->dimension(2);
+
+ _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type());
+ _is_prepared = false;
+ _original_weights = weights_to_use;
+
+ // Should bias be appended ?
+ bool append_bias = (biases != nullptr) && !_is_quantized;
+
+ // Calculate output shape
+ TensorShape output_shape = shape_calculator::compute_depthwise_convolution_shape(*input->info(), *weights->info(), conv_info, depth_multiplier, dilation);
+
+ // Output auto inizialitation if not yet initialized
+ auto_init_if_empty(*output->info(), input->info()->clone()->set_tensor_shape(output_shape));
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_DIMENSIONS(output->info()->tensor_shape(), output_shape);
+
+ if(_is_nhwc)
+ {
+ permute(output_shape, PermutationVector(1U, 2U, 0U));
+ _permuted_output.allocator()->init(output->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape));
+ _permuted_output.info()->set_data_layout(DataLayout::NCHW);
+ _permuted_output.info()->set_quantization_info(output->info()->quantization_info());
+ output_to_use = &_permuted_output;
+ }
+
+ // Output width and height
+ const unsigned int conv_w = output_shape.x();
+ const unsigned int conv_h = output_shape.y();
+
+ // Set up intermediate tensors
+ const size_t patch_size = weights_w * weights_h + (append_bias ? 1 : 0);
+ const size_t conv_size = conv_w * conv_h;
+
+ // Im2Col configuration
+ TensorShape shape_im2col = input_to_use->info()->tensor_shape();
+ shape_im2col.set(0, patch_size);
+ shape_im2col.set(1, conv_size);
+ shape_im2col.set(2, weights_z);
+ _input_reshaped.allocator()->init(input->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_im2col).set_data_layout(DataLayout::NCHW));
+ _im2col_kernel.configure(input_to_use, &_input_reshaped, Size2D(weights_w, weights_h), conv_info, append_bias, depth_multiplier, dilation);
+
+ // Weights reshape configuration
+ const TensorShape shape_weights_reshape(patch_size, weights_z);
+ _weights_reshaped.allocator()->init(weights->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_weights_reshape).set_data_layout(DataLayout::NCHW));
+ _weights_reshape_kernel.configure(weights_to_use, &_weights_reshaped, append_bias ? biases : nullptr);
+
+ // GEMV configuration
+ DataType v2mm_dt = (input->info()->data_type() == DataType::QASYMM8) ? DataType::S32 : input->info()->data_type();
+ TensorShape shape_v2mm_out = input_to_use->info()->tensor_shape();
+ shape_v2mm_out.set(0, conv_size * weights_z);
+ shape_v2mm_out.set(1, 1);
+ shape_v2mm_out.set(2, 1);
+ _v2mm_output.allocator()->init(input->info()->clone()->set_is_resizable(true).reset_padding().set_data_type(v2mm_dt).set_tensor_shape(shape_v2mm_out).set_data_layout(DataLayout::NCHW));
+ _v2mm_kernel.configure(&_input_reshaped, &_weights_reshaped, &_v2mm_output);
+ _output_reshaped.allocator()->init(_v2mm_output.info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape));
+ _vector_to_tensor_kernel.configure(&_v2mm_output, (_is_quantized) ? &_output_reshaped : output_to_use, conv_w, conv_h);
+
+ // Output staged configuration
+ if(_is_quantized)
+ {
+ const UniformQuantizationInfo iq_info = input->info()->quantization_info().uniform();
+ const UniformQuantizationInfo wq_info = weights->info()->quantization_info().uniform();
+ const UniformQuantizationInfo oq_info = output->info()->quantization_info().uniform();
+
+ float multiplier = (iq_info.scale * wq_info.scale) / oq_info.scale;
+ int output_multiplier;
+ int output_shift;
+ quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier, &output_shift);
+ _output_stage_kernel.configure(&_output_reshaped, biases, output_to_use, output_multiplier, output_shift, oq_info.offset);
+ _output_reshaped.allocator()->allocate();
+ }
+
+ if(_is_nhwc)
+ {
+ _permute_output.configure(&_permuted_output, output, PermutationVector(2U, 0U, 1U));
+
+ _permuted_input.allocator()->allocate();
+ _permuted_weights.allocator()->allocate();
+ _permuted_output.allocator()->allocate();
+ }
+
+ // Fill borders on inputs
+ PixelValue zero_in(static_cast<int32_t>(0));
+ PixelValue zero_w(static_cast<int32_t>(0));
+ if(_is_quantized)
+ {
+ zero_in = PixelValue(static_cast<int32_t>(input->info()->quantization_info().uniform().offset));
+ zero_w = PixelValue(static_cast<int32_t>(weights->info()->quantization_info().uniform().offset));
+ }
+ BorderSize border_size = _v2mm_kernel.border_size();
+ _v2mm_input_fill_border.configure(&_input_reshaped, border_size, BorderMode::CONSTANT, zero_in);
+
+ border_size.bottom = 0;
+ _v2mm_weights_fill_border.configure(&_weights_reshaped, border_size, BorderMode::CONSTANT, zero_w);
+
+ // Allocate intermediate tensors
+ _input_reshaped.allocator()->allocate();
+ _v2mm_output.allocator()->allocate();
}
-
- const size_t weights_w = weights_to_use->info()->dimension(0);
- const size_t weights_h = weights_to_use->info()->dimension(1);
- const size_t weights_z = weights_to_use->info()->dimension(2);
-
- _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type());
- _is_prepared = false;
- _original_weights = weights_to_use;
-
- // Should bias be appended ?
- bool append_bias = (biases != nullptr) && !_is_quantized;
-
- // Calculate output shape
- TensorShape output_shape = shape_calculator::compute_depthwise_convolution_shape(*input->info(), *weights->info(), conv_info, depth_multiplier, dilation);
-
- // Output auto inizialitation if not yet initialized
- auto_init_if_empty(*output->info(), input->info()->clone()->set_tensor_shape(output_shape));
- ARM_COMPUTE_ERROR_ON_MISMATCHING_DIMENSIONS(output->info()->tensor_shape(), output_shape);
-
- if(_is_nhwc)
+ else
{
- permute(output_shape, PermutationVector(1U, 2U, 0U));
- _permuted_output.allocator()->init(output->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape));
- _permuted_output.info()->set_data_layout(DataLayout::NCHW);
- _permuted_output.info()->set_quantization_info(output->info()->quantization_info());
- output_to_use = &_permuted_output;
+ // Configure kernel
+ _depthwise_conv_kernel.configure(input, weights, biases, output, conv_info, depth_multiplier, dilation);
+
+ // Fill input borders
+ _fill_border.configure(input, _depthwise_conv_kernel.border_size(), BorderMode::CONSTANT, PixelValue(static_cast<uint64_t>(0), input->info()->data_type()));
}
- // Output width and height
- const unsigned int conv_w = output_shape.x();
- const unsigned int conv_h = output_shape.y();
-
- // Set up intermediate tensors
- const size_t patch_size = weights_w * weights_h + (append_bias ? 1 : 0);
- const size_t conv_size = conv_w * conv_h;
-
- // Im2Col configuration
- TensorShape shape_im2col = input_to_use->info()->tensor_shape();
- shape_im2col.set(0, patch_size);
- shape_im2col.set(1, conv_size);
- shape_im2col.set(2, weights_z);
- _input_reshaped.allocator()->init(input->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_im2col).set_data_layout(DataLayout::NCHW));
- _im2col_kernel.configure(input_to_use, &_input_reshaped, Size2D(weights_w, weights_h), conv_info, append_bias, depth_multiplier, dilation);
-
- // Weights reshape configuration
- const TensorShape shape_weights_reshape(patch_size, weights_z);
- _weights_reshaped.allocator()->init(weights->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_weights_reshape).set_data_layout(DataLayout::NCHW));
- _weights_reshape_kernel.configure(weights_to_use, &_weights_reshaped, append_bias ? biases : nullptr);
-
- // GEMV configuration
- DataType v2mm_dt = (input->info()->data_type() == DataType::QASYMM8) ? DataType::S32 : input->info()->data_type();
- TensorShape shape_v2mm_out = input_to_use->info()->tensor_shape();
- shape_v2mm_out.set(0, conv_size * weights_z);
- shape_v2mm_out.set(1, 1);
- shape_v2mm_out.set(2, 1);
- _v2mm_output.allocator()->init(input->info()->clone()->set_is_resizable(true).reset_padding().set_data_type(v2mm_dt).set_tensor_shape(shape_v2mm_out).set_data_layout(DataLayout::NCHW));
- _v2mm_kernel.configure(&_input_reshaped, &_weights_reshaped, &_v2mm_output);
- _output_reshaped.allocator()->init(_v2mm_output.info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape));
- _vector_to_tensor_kernel.configure(&_v2mm_output, (_is_quantized) ? &_output_reshaped : output_to_use, conv_w, conv_h);
-
- // Output staged configuration
- if(_is_quantized)
- {
- const UniformQuantizationInfo iq_info = input->info()->quantization_info().uniform();
- const UniformQuantizationInfo wq_info = weights->info()->quantization_info().uniform();
- const UniformQuantizationInfo oq_info = output->info()->quantization_info().uniform();
-
- float multiplier = (iq_info.scale * wq_info.scale) / oq_info.scale;
- int output_multiplier;
- int output_shift;
- quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier, &output_shift);
- _output_stage_kernel.configure(&_output_reshaped, biases, output_to_use, output_multiplier, output_shift, oq_info.offset);
- _output_reshaped.allocator()->allocate();
- }
-
- if(_is_nhwc)
- {
- _permute_output.configure(&_permuted_output, output, PermutationVector(2U, 0U, 1U));
-
- _permuted_input.allocator()->allocate();
- _permuted_weights.allocator()->allocate();
- _permuted_output.allocator()->allocate();
- }
-
- // Fill borders on inputs
- PixelValue zero_in(static_cast<int32_t>(0));
- PixelValue zero_w(static_cast<int32_t>(0));
- if(_is_quantized)
- {
- zero_in = PixelValue(static_cast<int32_t>(input->info()->quantization_info().uniform().offset));
- zero_w = PixelValue(static_cast<int32_t>(weights->info()->quantization_info().uniform().offset));
- }
- BorderSize border_size = _v2mm_kernel.border_size();
- _v2mm_input_fill_border.configure(&_input_reshaped, border_size, BorderMode::CONSTANT, zero_in);
-
- border_size.bottom = 0;
- _v2mm_weights_fill_border.configure(&_weights_reshaped, border_size, BorderMode::CONSTANT, zero_w);
-
- // Allocate intermediate tensors
- _input_reshaped.allocator()->allocate();
- _v2mm_output.allocator()->allocate();
-
//Configure Activation Layer
_is_activationlayer_enabled = act_info.enabled();
@@ -845,89 +858,96 @@
ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(height_idx) + (weights->dimension(height_idx) - 1) * (dilation.y() - 1) > input->dimension(height_idx) + conv_info.pad_top() + conv_info.pad_bottom());
ARM_COMPUTE_RETURN_ERROR_ON((input->dimension(channel_idx) * depth_multiplier) != weights->dimension(channel_idx));
- // Clone output to use auto init
- auto output_clone = output->clone();
-
- const ITensorInfo *input_to_use = input;
- const ITensorInfo *weights_to_use = weights;
- const ITensorInfo *output_to_use = output_clone.get();
-
- TensorShape permuted_input_shape = input->tensor_shape();
- TensorShape permuted_weights_shape = weights->tensor_shape();
- TensorInfo permuted_input;
- TensorInfo permuted_weights;
-
- if(input->data_layout() == DataLayout::NHWC)
+ if(input->data_layout() != DataLayout::NHWC || input->data_type() != DataType::F32)
{
- permute(permuted_input_shape, PermutationVector(1U, 2U, 0U));
- permute(permuted_weights_shape, PermutationVector(1U, 2U, 0U));
+ // Clone output to use auto init
+ auto output_clone = output->clone();
- permuted_input = TensorInfo(input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(permuted_input_shape).set_data_layout(DataLayout::NCHW));
- permuted_weights = TensorInfo(weights->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(permuted_weights_shape).set_data_layout(DataLayout::NCHW));
+ const ITensorInfo *input_to_use = input;
+ const ITensorInfo *weights_to_use = weights;
+ const ITensorInfo *output_to_use = output_clone.get();
- input_to_use = &permuted_input;
- weights_to_use = &permuted_weights;
+ TensorShape permuted_input_shape = input->tensor_shape();
+ TensorShape permuted_weights_shape = weights->tensor_shape();
+ TensorInfo permuted_input;
+ TensorInfo permuted_weights;
+
+ if(input->data_layout() == DataLayout::NHWC)
+ {
+ permute(permuted_input_shape, PermutationVector(1U, 2U, 0U));
+ permute(permuted_weights_shape, PermutationVector(1U, 2U, 0U));
+
+ permuted_input = TensorInfo(input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(permuted_input_shape).set_data_layout(DataLayout::NCHW));
+ permuted_weights = TensorInfo(weights->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(permuted_weights_shape).set_data_layout(DataLayout::NCHW));
+
+ input_to_use = &permuted_input;
+ weights_to_use = &permuted_weights;
+ }
+
+ const bool is_quantized = is_data_type_quantized_asymmetric(input->data_type());
+ const bool append_bias = (biases != nullptr) && !is_quantized;
+ TensorShape output_shape = shape_calculator::compute_depthwise_convolution_shape(*input, *weights, conv_info, depth_multiplier, dilation);
+ const size_t weights_w = weights_to_use->dimension(0);
+ const size_t weights_h = weights_to_use->dimension(1);
+ const size_t weights_z = weights_to_use->dimension(2);
+ const unsigned int conv_w = output_shape[width_idx];
+ const unsigned int conv_h = output_shape[height_idx];
+ const size_t patch_size = weights_w * weights_h + (append_bias ? 1 : 0);
+ const size_t conv_size = conv_w * conv_h;
+
+ // Output auto inizialitation if not yet initialized
+ auto_init_if_empty(*output_clone, input->clone()->set_tensor_shape(output_shape));
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(output->tensor_shape(), output_shape);
+
+ TensorInfo permuted_output;
+ if(input->data_layout() == DataLayout::NHWC)
+ {
+ permute(output_shape, PermutationVector(1U, 2U, 0U));
+ permuted_output = TensorInfo(output_clone->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape).set_data_layout(DataLayout::NCHW));
+ output_to_use = &permuted_output;
+ }
+
+ // Im2Col configuration
+ TensorShape shape_im2col = input_to_use->tensor_shape();
+ shape_im2col.set(0, patch_size);
+ shape_im2col.set(1, conv_size);
+ shape_im2col.set(2, weights_z);
+ TensorInfo input_reshaped(input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_im2col).set_data_layout(DataLayout::NCHW));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEDepthwiseIm2ColKernel::validate(input_to_use, &input_reshaped, Size2D(weights_w, weights_h), conv_info, append_bias, depth_multiplier, dilation));
+
+ // Weights reshape configuration
+ const TensorShape shape_weights_reshape(patch_size, weights_z);
+ TensorInfo weights_reshaped(weights->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_weights_reshape).set_data_layout(DataLayout::NCHW));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEDepthwiseWeightsReshapeKernel::validate(weights_to_use, &weights_reshaped, append_bias ? biases : nullptr));
+
+ // GEMV configuration
+ DataType v2mm_dt = (input->data_type() == DataType::QASYMM8) ? DataType::S32 : input->data_type();
+ TensorShape shape_v2mm_out = input_to_use->tensor_shape();
+ shape_v2mm_out.set(0, conv_size * weights_z);
+ shape_v2mm_out.set(1, 1);
+ shape_v2mm_out.set(2, 1);
+ TensorInfo v2mm_output(input->clone()->set_is_resizable(true).reset_padding().set_data_type(v2mm_dt).set_tensor_shape(shape_v2mm_out).set_data_layout(DataLayout::NCHW));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMMatrixVectorMultiplyKernel::validate(&input_reshaped, &weights_reshaped, &v2mm_output));
+
+ TensorInfo output_reshaped(v2mm_output.clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_to_use->tensor_shape()));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEDepthwiseVectorToTensorKernel::validate(&v2mm_output, (is_quantized) ? &output_reshaped : output_to_use, conv_w, conv_h));
+
+ if(is_quantized)
+ {
+ const UniformQuantizationInfo iq_info = input->quantization_info().uniform();
+ const UniformQuantizationInfo wq_info = weights->quantization_info().uniform();
+ const UniformQuantizationInfo oq_info = output->quantization_info().uniform();
+
+ float multiplier = (iq_info.scale * wq_info.scale) / oq_info.scale;
+ int output_multiplier;
+ int output_shift;
+ ARM_COMPUTE_RETURN_ON_ERROR(quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier, &output_shift));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEDirectConvolutionLayerOutputStageKernel::validate(&output_reshaped, biases, output_to_use, output_multiplier, output_shift, oq_info.offset));
+ }
}
-
- const bool is_quantized = is_data_type_quantized_asymmetric(input->data_type());
- const bool append_bias = (biases != nullptr) && !is_quantized;
- TensorShape output_shape = shape_calculator::compute_depthwise_convolution_shape(*input, *weights, conv_info, depth_multiplier, dilation);
- const size_t weights_w = weights_to_use->dimension(0);
- const size_t weights_h = weights_to_use->dimension(1);
- const size_t weights_z = weights_to_use->dimension(2);
- const unsigned int conv_w = output_shape[width_idx];
- const unsigned int conv_h = output_shape[height_idx];
- const size_t patch_size = weights_w * weights_h + (append_bias ? 1 : 0);
- const size_t conv_size = conv_w * conv_h;
-
- // Output auto inizialitation if not yet initialized
- auto_init_if_empty(*output_clone, input->clone()->set_tensor_shape(output_shape));
- ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(output->tensor_shape(), output_shape);
-
- TensorInfo permuted_output;
- if(input->data_layout() == DataLayout::NHWC)
+ else
{
- permute(output_shape, PermutationVector(1U, 2U, 0U));
- permuted_output = TensorInfo(output_clone->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape).set_data_layout(DataLayout::NCHW));
- output_to_use = &permuted_output;
- }
-
- // Im2Col configuration
- TensorShape shape_im2col = input_to_use->tensor_shape();
- shape_im2col.set(0, patch_size);
- shape_im2col.set(1, conv_size);
- shape_im2col.set(2, weights_z);
- TensorInfo input_reshaped(input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_im2col).set_data_layout(DataLayout::NCHW));
- ARM_COMPUTE_RETURN_ON_ERROR(NEDepthwiseIm2ColKernel::validate(input_to_use, &input_reshaped, Size2D(weights_w, weights_h), conv_info, append_bias, depth_multiplier, dilation));
-
- // Weights reshape configuration
- const TensorShape shape_weights_reshape(patch_size, weights_z);
- TensorInfo weights_reshaped(weights->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_weights_reshape).set_data_layout(DataLayout::NCHW));
- ARM_COMPUTE_RETURN_ON_ERROR(NEDepthwiseWeightsReshapeKernel::validate(weights_to_use, &weights_reshaped, append_bias ? biases : nullptr));
-
- // GEMV configuration
- DataType v2mm_dt = (input->data_type() == DataType::QASYMM8) ? DataType::S32 : input->data_type();
- TensorShape shape_v2mm_out = input_to_use->tensor_shape();
- shape_v2mm_out.set(0, conv_size * weights_z);
- shape_v2mm_out.set(1, 1);
- shape_v2mm_out.set(2, 1);
- TensorInfo v2mm_output(input->clone()->set_is_resizable(true).reset_padding().set_data_type(v2mm_dt).set_tensor_shape(shape_v2mm_out).set_data_layout(DataLayout::NCHW));
- ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMMatrixVectorMultiplyKernel::validate(&input_reshaped, &weights_reshaped, &v2mm_output));
-
- TensorInfo output_reshaped(v2mm_output.clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_to_use->tensor_shape()));
- ARM_COMPUTE_RETURN_ON_ERROR(NEDepthwiseVectorToTensorKernel::validate(&v2mm_output, (is_quantized) ? &output_reshaped : output_to_use, conv_w, conv_h));
-
- if(is_quantized)
- {
- const UniformQuantizationInfo iq_info = input->quantization_info().uniform();
- const UniformQuantizationInfo wq_info = weights->quantization_info().uniform();
- const UniformQuantizationInfo oq_info = output->quantization_info().uniform();
-
- float multiplier = (iq_info.scale * wq_info.scale) / oq_info.scale;
- int output_multiplier;
- int output_shift;
- ARM_COMPUTE_RETURN_ON_ERROR(quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier, &output_shift));
- ARM_COMPUTE_RETURN_ON_ERROR(NEDirectConvolutionLayerOutputStageKernel::validate(&output_reshaped, biases, output_to_use, output_multiplier, output_shift, oq_info.offset));
+ ARM_COMPUTE_RETURN_ON_ERROR(NEDepthwiseConvolutionLayerKernel::validate(input, weights, biases, output, conv_info, depth_multiplier, dilation));
}
// Validate Activation Layer
@@ -941,25 +961,33 @@
void NEDepthwiseConvolutionLayer::run()
{
- prepare();
-
- if(_is_nhwc)
+ if(!_is_optimized)
{
- _permute_input.run();
+ prepare();
+
+ if(_is_nhwc)
+ {
+ _permute_input.run();
+ }
+
+ NEScheduler::get().schedule(&_im2col_kernel, Window::DimX);
+ NEScheduler::get().schedule(&_v2mm_input_fill_border, Window::DimX);
+ NEScheduler::get().schedule(&_v2mm_kernel, Window::DimX);
+ NEScheduler::get().schedule(&_vector_to_tensor_kernel, Window::DimX);
+ if(_is_quantized)
+ {
+ NEScheduler::get().schedule(&_output_stage_kernel, Window::DimX);
+ }
+
+ if(_is_nhwc)
+ {
+ _permute_output.run();
+ }
}
-
- NEScheduler::get().schedule(&_im2col_kernel, Window::DimX);
- NEScheduler::get().schedule(&_v2mm_input_fill_border, Window::DimX);
- NEScheduler::get().schedule(&_v2mm_kernel, Window::DimX);
- NEScheduler::get().schedule(&_vector_to_tensor_kernel, Window::DimX);
- if(_is_quantized)
+ else
{
- NEScheduler::get().schedule(&_output_stage_kernel, Window::DimX);
- }
-
- if(_is_nhwc)
- {
- _permute_output.run();
+ NEScheduler::get().schedule(&_fill_border, Window::DimX);
+ NEScheduler::get().schedule(&_depthwise_conv_kernel, Window::DimY);
}
if(_is_activationlayer_enabled)
@@ -970,7 +998,7 @@
void NEDepthwiseConvolutionLayer::prepare()
{
- if(!_is_prepared)
+ if(!_is_prepared && !_is_optimized)
{
ARM_COMPUTE_ERROR_ON(!_original_weights->is_used());
diff --git a/tests/NEON/Helper.h b/tests/NEON/Helper.h
index c30cbc9..7446e5a 100644
--- a/tests/NEON/Helper.h
+++ b/tests/NEON/Helper.h
@@ -88,6 +88,26 @@
}
};
+/** As above but this also setups a Zero border on the input tensor of the kernel's bordersize */
+template <typename K>
+class NESynthetizeFunctionWithZeroConstantKernelBorder : public INESimpleFunction
+{
+public:
+ /** Configure the kernel.
+ *
+ * @param[in] first First configuration argument.
+ * @param[in] args Rest of the configuration arguments.
+ */
+ template <typename T, typename... Args>
+ void configure(T first, Args &&... args)
+ {
+ auto k = arm_compute::support::cpp14::make_unique<K>();
+ k->configure(first, std::forward<Args>(args)...);
+ _kernel = std::move(k);
+ _border_handler.configure(first, BorderSize(_kernel->border_size()), BorderMode::CONSTANT, PixelValue());
+ }
+};
+
} // namespace test
} // namespace arm_compute
#endif /* __ARM_COMPUTE_TEST_NEON_HELPER_H__ */
diff --git a/tests/validation/NEON/DepthwiseConvolutionLayerKernel.cpp b/tests/validation/NEON/DepthwiseConvolutionLayerKernel.cpp
new file mode 100644
index 0000000..3af8358
--- /dev/null
+++ b/tests/validation/NEON/DepthwiseConvolutionLayerKernel.cpp
@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "arm_compute/core/NEON/kernels/NEDepthwiseConvolutionLayerKernel.h"
+#include "tests/NEON/Accessor.h"
+#include "tests/NEON/Helper.h"
+#include "tests/framework/Macros.h"
+#include "tests/framework/datasets/Datasets.h"
+#include "tests/validation/Validation.h"
+#include "tests/validation/fixtures/DepthwiseConvolutionLayerFixture.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace validation
+{
+using namespace arm_compute::misc::shape_calculator;
+
+// Create function for NEDepthwiseConvolutionLayerKernel
+using NEDepthwiseConvolutionLayer = NESynthetizeFunctionWithZeroConstantKernelBorder<NEDepthwiseConvolutionLayerKernel>;
+
+// Fixture for NEDepthwiseConvolutionLayerKernel
+template <typename T>
+using NEDepthwiseConvolutionLayerKernelFixture = DepthwiseConvolutionLayerKernelValidationFixture<Tensor, Accessor, NEDepthwiseConvolutionLayer, T>;
+
+namespace
+{
+// *INDENT-OFF*
+// clang-format off
+RelativeTolerance<float> rel_tolerance_f32(0.001f);
+constexpr float abs_tolerance_f32(0.0001f);
+
+/** Width values to test - Precommit */
+const auto width_values = framework::dataset::make("width", { 17U, 47U } );
+
+/** Height values to test - Precommit */
+const auto height_values = framework::dataset::make("height", { 19U, 43U } );
+
+/** Channel values to test - Precommit */
+const auto channel_values = framework::dataset::make("channels", { 32U, 128U });
+
+/** Batch values to test - Precommit */
+const auto batch_values = framework::dataset::make("batch", { 1U, 3U });
+
+/** Kernel size values to test - Precommit */
+const auto kernel_sz_values = framework::dataset::make("kernel_size", { Size2D(3U, 5U), Size2D(5U, 3U) });
+
+/** Depth multiplier values to test - Precommit */
+const auto depth_multiplier_values = framework::dataset::make("depth_multiplier", { 1U, 3U });
+
+/** Dilation values to test - Precommit */
+const auto dilation_values = framework::dataset::make("dilation", { Size2D(1U, 1U), Size2D(3U, 3U) });
+
+/** Stride values to test - All */
+const auto stride_values = framework::dataset::make("stride", { Size2D(1U, 1U), Size2D(3U, 2U) });
+
+/** Padding values to test - All */
+const auto padding_valid_values = framework::dataset::make("padding_valid", { true, false });
+
+/** Data type values to test - All */
+const auto data_type_values = framework::dataset::make("data_type", { DataType::F32 });
+
+/** Data layout values to test - All */
+const auto data_layout_values = framework::dataset::make("data_layout", { DataLayout::NHWC });
+
+/** Configuration test */
+void validate_configuration(size_t width_value, size_t height_value, size_t channel_value, size_t batch_value, Size2D kernel_sz_value, size_t depth_multiplier_value, Size2D dilation_value, Size2D stride_value, bool padding_valid_value, DataType data_type_value, DataLayout data_layout_value)
+{
+ TensorShape src_shape(width_value, height_value, channel_value, batch_value);
+ TensorShape weights_shape(kernel_sz_value.width, kernel_sz_value.height, channel_value * depth_multiplier_value);
+ TensorShape biases_shape(channel_value * depth_multiplier_value);
+
+ if(data_layout_value == DataLayout::NHWC)
+ {
+ permute(src_shape, PermutationVector(2U, 0U, 1U, 3U));
+ permute(weights_shape, PermutationVector(2U, 0U, 1U));
+ }
+
+ TensorInfo src_info(src_shape, 1, data_type_value);
+ TensorInfo weights_info(weights_shape, 1, data_type_value);
+ TensorInfo biases_info(biases_shape, 1, data_type_value);
+
+ src_info.set_data_layout(data_layout_value);
+ weights_info.set_data_layout(data_layout_value);
+ biases_info.set_data_layout(data_layout_value);
+
+ PadStrideInfo conv_info;
+ if(padding_valid_value)
+ {
+ conv_info = PadStrideInfo();
+ }
+ else
+ {
+ conv_info = calculate_same_pad(src_shape, weights_shape, PadStrideInfo(stride_value.width, stride_value.height), data_layout_value, dilation_value);
+ }
+
+ const TensorShape dst_shape = compute_depthwise_convolution_shape(src_info, weights_info, conv_info, depth_multiplier_value, dilation_value);
+
+ // Create tensors
+ Tensor src = create_tensor<Tensor>(src_shape, data_type_value, 1, QuantizationInfo(), data_layout_value);
+ Tensor weights = create_tensor<Tensor>(weights_shape, data_type_value, 1, QuantizationInfo(), data_layout_value);
+ Tensor biases = create_tensor<Tensor>(biases_shape, data_type_value, 1, QuantizationInfo(), data_layout_value);
+ Tensor dst = create_tensor<Tensor>(dst_shape, data_type_value, 1, QuantizationInfo(), data_layout_value);
+
+ ARM_COMPUTE_EXPECT(src.info()->is_resizable(), framework::LogLevel::ERRORS);
+ ARM_COMPUTE_EXPECT(weights.info()->is_resizable(), framework::LogLevel::ERRORS);
+ ARM_COMPUTE_EXPECT(biases.info()->is_resizable(), framework::LogLevel::ERRORS);
+ ARM_COMPUTE_EXPECT(dst.info()->is_resizable(), framework::LogLevel::ERRORS);
+
+ // Create and configure function
+ NEDepthwiseConvolutionLayer dwc;
+ dwc.configure(&src, &weights, &biases, &dst, conv_info, depth_multiplier_value, dilation_value);
+}
+} // namespace
+
+TEST_SUITE(NEON)
+TEST_SUITE(DepthwiseConvolutionLayer)
+TEST_SUITE(Float)
+TEST_SUITE(FP32)
+DATA_TEST_CASE(Configuration, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(combine(combine(combine(width_values,
+ height_values),
+ channel_values),
+ batch_values),
+ kernel_sz_values),
+ depth_multiplier_values),
+ dilation_values),
+ stride_values),
+ padding_valid_values),
+ data_type_values),
+ data_layout_values),
+width_value, height_value, channel_value, batch_value, kernel_sz_value, depth_multiplier_value, dilation_value, stride_value, padding_valid_value, data_type_value, data_layout_value)
+{
+ validate_configuration(width_value, height_value, channel_value, batch_value, kernel_sz_value, depth_multiplier_value, dilation_value, stride_value, padding_valid_value, data_type_value, data_layout_value);
+}
+
+FIXTURE_DATA_TEST_CASE(RunSmall, NEDepthwiseConvolutionLayerKernelFixture<float>, framework::DatasetMode::ALL,
+ combine(combine(combine(combine(combine(combine(combine(combine(combine(combine(width_values,
+ height_values),
+ channel_values),
+ batch_values),
+ kernel_sz_values),
+ depth_multiplier_values),
+ dilation_values),
+ stride_values),
+ padding_valid_values),
+ data_type_values),
+ data_layout_values))
+{
+ // Validate output
+ validate(Accessor(_target), _reference, rel_tolerance_f32, 0.f, abs_tolerance_f32);
+}
+
+TEST_SUITE_END() // FP32
+TEST_SUITE_END() // Float
+TEST_SUITE_END() // DepthwiseConvolutionLayer
+TEST_SUITE_END() // NEON
+} // namespace validation
+} // namespace test
+} // namespace arm_compute
\ No newline at end of file
diff --git a/tests/validation/fixtures/DepthwiseConvolutionLayerFixture.h b/tests/validation/fixtures/DepthwiseConvolutionLayerFixture.h
index b01e176..30b8df9 100644
--- a/tests/validation/fixtures/DepthwiseConvolutionLayerFixture.h
+++ b/tests/validation/fixtures/DepthwiseConvolutionLayerFixture.h
@@ -193,6 +193,115 @@
};
template <typename TensorType, typename AccessorType, typename FunctionType, typename T>
+class DepthwiseConvolutionLayerKernelValidationFixture : public DepthwiseConvolutionLayerValidationGenericFixture<TensorType, AccessorType, FunctionType, T>
+{
+public:
+ template <typename...>
+ void setup(size_t width, size_t height, size_t channel, size_t batch, Size2D kernel_size, size_t depth_multiplier, Size2D dilation, Size2D stride, bool padding_valid, DataType data_type,
+ DataLayout data_layout)
+ {
+ const TensorShape src_shape(width, height, channel, batch);
+ const TensorShape weights_shape(kernel_size.width, kernel_size.height, channel * depth_multiplier);
+ const TensorShape biases_shape(weights_shape.z());
+
+ PadStrideInfo conv_info;
+ if(padding_valid)
+ {
+ conv_info = PadStrideInfo();
+ }
+ else
+ {
+ conv_info = calculate_same_pad(src_shape, weights_shape, PadStrideInfo(stride.width, stride.height), DataLayout::NCHW, dilation);
+ }
+
+ _target = compute_target(src_shape, weights_shape, biases_shape, conv_info, dilation, depth_multiplier, data_type, data_layout);
+ _reference = compute_reference(src_shape, weights_shape, biases_shape, conv_info, dilation, depth_multiplier, data_type);
+ }
+
+protected:
+ template <typename U>
+ void fill(U &&tensor, int i)
+ {
+ switch(tensor.data_type())
+ {
+ case DataType::F32:
+ {
+ std::uniform_real_distribution<> distribution(-1.0f, 1.0f);
+ library->fill(tensor, distribution, i);
+ break;
+ }
+ default:
+ library->fill_tensor_uniform(tensor, i);
+ }
+ }
+
+ TensorType compute_target(TensorShape input_shape, TensorShape weights_shape, TensorShape biases_shape, PadStrideInfo &conv_info, Size2D dilation,
+ unsigned int depth_multiplier, const DataType data_type, const DataLayout data_layout)
+ {
+ if(data_layout == DataLayout::NHWC)
+ {
+ permute(input_shape, PermutationVector(2U, 0U, 1U));
+ permute(weights_shape, PermutationVector(2U, 0U, 1U));
+ }
+
+ // Create tensors
+ TensorType src = create_tensor<TensorType>(input_shape, data_type, 1, QuantizationInfo(), data_layout);
+ TensorType weights = create_tensor<TensorType>(weights_shape, data_type, 1, QuantizationInfo(), data_layout);
+ TensorType biases = create_tensor<TensorType>(biases_shape, data_type, 1, QuantizationInfo(), data_layout);
+ TensorType dst = create_tensor<TensorType>(TensorShape(), data_type, 1, QuantizationInfo(), data_layout);
+
+ // Create Depthwise Convolution configure function
+ FunctionType dwc;
+ dwc.configure(&src, &weights, &biases, &dst, conv_info, depth_multiplier, dilation);
+
+ ARM_COMPUTE_EXPECT(src.info()->is_resizable(), framework::LogLevel::ERRORS);
+ ARM_COMPUTE_EXPECT(weights.info()->is_resizable(), framework::LogLevel::ERRORS);
+ ARM_COMPUTE_EXPECT(biases.info()->is_resizable(), framework::LogLevel::ERRORS);
+ ARM_COMPUTE_EXPECT(dst.info()->is_resizable(), framework::LogLevel::ERRORS);
+
+ // Allocate tensors
+ src.allocator()->allocate();
+ weights.allocator()->allocate();
+ biases.allocator()->allocate();
+ dst.allocator()->allocate();
+
+ ARM_COMPUTE_EXPECT(!src.info()->is_resizable(), framework::LogLevel::ERRORS);
+ ARM_COMPUTE_EXPECT(!weights.info()->is_resizable(), framework::LogLevel::ERRORS);
+ ARM_COMPUTE_EXPECT(!biases.info()->is_resizable(), framework::LogLevel::ERRORS);
+ ARM_COMPUTE_EXPECT(!dst.info()->is_resizable(), framework::LogLevel::ERRORS);
+
+ // Fill tensors
+ fill(AccessorType(src), 0);
+ fill(AccessorType(weights), 1);
+ fill(AccessorType(biases), 2);
+
+ // Compute function
+ dwc.run();
+
+ return dst;
+ }
+
+ SimpleTensor<T> compute_reference(const TensorShape &input_shape, const TensorShape &weights_shape, const TensorShape &biases_shape, const PadStrideInfo &conv_info,
+ const Size2D &dilation, unsigned int depth_multiplier, const DataType data_type)
+ {
+ SimpleTensor<T> src{ input_shape, data_type };
+ SimpleTensor<T> weights{ weights_shape, data_type };
+ SimpleTensor<T> biases{ biases_shape, data_type };
+
+ fill(src, 0);
+ fill(weights, 1);
+ fill(biases, 2);
+
+ const TensorShape dst_shape = compute_depthwise_convolution_shape(TensorInfo(input_shape, 1, data_type), TensorInfo(weights_shape, 1, data_type), conv_info,
+ depth_multiplier, dilation);
+ return reference::depthwise_convolution(src, weights, biases, dst_shape, conv_info, depth_multiplier, dilation);
+ }
+
+ TensorType _target{};
+ SimpleTensor<T> _reference{};
+};
+
+template <typename TensorType, typename AccessorType, typename FunctionType, typename T>
class DepthwiseConvolutionLayerValidationQuantizedFixture : public DepthwiseConvolutionLayerValidationGenericFixture<TensorType, AccessorType, FunctionType, T>
{
public: