Implement CLDirectConv3D f32/f16
Resolve COMPMID-4660
Signed-off-by: Giorgio Arena <giorgio.arena@arm.com>
Change-Id: Ibd66ec1eb6faa60086981b1e3a9c12561df3445f
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/6420
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
diff --git a/src/gpu/cl/kernels/ClDirectConv2dKernel.cpp b/src/gpu/cl/kernels/ClDirectConv2dKernel.cpp
index cbeb9c4..2d851a6 100644
--- a/src/gpu/cl/kernels/ClDirectConv2dKernel.cpp
+++ b/src/gpu/cl/kernels/ClDirectConv2dKernel.cpp
@@ -60,19 +60,17 @@
const int height_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
const int channel_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::CHANNEL);
- ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->dimension(width_idx) != weights->dimension(height_idx), "Weights should have same width and height");
- ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->dimension(channel_idx) != src->dimension(channel_idx),
- "Weights feature map dimension should match the respective src's one");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->dimension(channel_idx) != src->dimension(channel_idx), "Weights feature map dimension should match the respective src's one");
ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->num_dimensions() > 4, "Weights can be at most 4 dimensional");
- ARM_COMPUTE_RETURN_ERROR_ON_MSG((weights->dimension(width_idx) == 1) && std::get<0>(conv_info.stride()) > 3, "Strides larger than 3 not supported for 1x1 convolution.");
- ARM_COMPUTE_RETURN_ERROR_ON_MSG((weights->dimension(width_idx) == 3 || weights->dimension(width_idx) == 5 || weights->dimension(width_idx) == 9)
- && std::get<0>(conv_info.stride()) > 2,
- "Strides larger than 2 not supported for 3x3, 5x5, 9x9 convolution.");
- ARM_COMPUTE_RETURN_ERROR_ON_MSG(data_layout != DataLayout::NHWC && !is_data_type_float(src->data_type()) && act_info.enabled(),
- "Activation supported only for floating point and NHWC.");
if(data_layout == DataLayout::NCHW)
{
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->dimension(width_idx) != weights->dimension(height_idx), "Weights should have same width and height");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG((weights->dimension(width_idx) == 1) && std::get<0>(conv_info.stride()) > 3, "Strides larger than 3 not supported for 1x1 convolution.");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG((weights->dimension(width_idx) == 3 || weights->dimension(width_idx) == 5 || weights->dimension(width_idx) == 9) && std::get<0>(conv_info.stride()) > 2,
+ "Strides larger than 2 not supported for 3x3, 5x5, 9x9 convolution.");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(!is_data_type_float(src->data_type()) && act_info.enabled(), "Activation supported only for floating point and NHWC.");
+
if(is_data_type_quantized(src->data_type()))
{
ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->dimension(width_idx) != 1 && weights->dimension(width_idx) != 3 && weights->dimension(width_idx) != 5 && weights->dimension(width_idx) != 9,
@@ -96,7 +94,7 @@
ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases);
}
ARM_COMPUTE_RETURN_ERROR_ON_MSG(biases->dimension(0) != weights->dimension(3),
- "Biases size and number of src feature maps should match");
+ "Biases size and number of dst feature maps should match");
ARM_COMPUTE_RETURN_ERROR_ON_MSG(biases->num_dimensions() > 1,
"Biases should be one dimensional");
}
diff --git a/src/gpu/cl/kernels/ClDirectConv2dKernel.h b/src/gpu/cl/kernels/ClDirectConv2dKernel.h
index 4041c7b..5624f3a 100644
--- a/src/gpu/cl/kernels/ClDirectConv2dKernel.h
+++ b/src/gpu/cl/kernels/ClDirectConv2dKernel.h
@@ -42,9 +42,9 @@
ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(ClDirectConv2dKernel);
/** Set the src, weights, biases and dst tensors info.
*
- * @note: Due to set_valid_region(), thus src/weights/biases cannot be const. Need to change this once the set_valid_region() is removed.
+ * @note: Due to set_valid_region() in NCHW, src/weights/biases cannot be const. Need to change this once the set_valid_region() is removed.
*
- * @note: DirectConvolution only works in the following configurations:
+ * @note: DirectConvolution only works in the following configurations for the NCHW data layout:
* 1x1 convolution with stride_x = 1/2/3, stride_y = 1/2/3
* 3x3 convolution with stride_x = 1/2, stride_y = 1/2
* 5x5 convolution with stride_x = 1/2, stride_y = 1/2
diff --git a/src/gpu/cl/kernels/ClDirectConv3dKernel.cpp b/src/gpu/cl/kernels/ClDirectConv3dKernel.cpp
new file mode 100644
index 0000000..1c4326b
--- /dev/null
+++ b/src/gpu/cl/kernels/ClDirectConv3dKernel.cpp
@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2021 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "src/gpu/cl/kernels/ClDirectConv3dKernel.h"
+
+#include "arm_compute/core/CL/ICLTensor.h"
+#include "arm_compute/core/utils/misc/ShapeCalculator.h"
+#include "src/core/CL/CLValidate.h"
+#include "src/core/helpers/WindowHelpers.h"
+#include "support/Cast.h"
+
+namespace arm_compute
+{
+namespace opencl
+{
+namespace kernels
+{
+namespace
+{
+Status validate_arguments(const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *dst, const Conv3dInfo &conv3d_info)
+{
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_LAYOUT(src, weights, dst);
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(src->data_layout() != DataLayout::NDHWC, "Only NDHWC layout supported");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(conv3d_info.act_info.enabled(), "Fused activation not supported");
+
+ ARM_COMPUTE_RETURN_ERROR_ON_F16_UNSUPPORTED(src);
+ ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(src, 1, DataType::F16, DataType::F32);
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(src, weights);
+
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->dimension(1) != src->dimension(0), "Weights feature map dimension should match the respective src's one");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights->num_dimensions() > 5, "Weights can be at most 5 dimensional");
+
+ ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(2) > (src->dimension(1) + conv3d_info.padding.left + conv3d_info.padding.right));
+ ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(3) > (src->dimension(2) + conv3d_info.padding.top + conv3d_info.padding.bottom));
+ ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(4) > (src->dimension(3) + conv3d_info.padding.front + conv3d_info.padding.back));
+
+ if(biases != nullptr)
+ {
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases);
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(biases->dimension(0) != weights->dimension(0), "Biases size and number of dst feature maps should match");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(biases->num_dimensions() > 1, "Biases should be one dimensional");
+ }
+
+ // Checks performed when dst is configured
+ if(dst->total_size() != 0)
+ {
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(dst->dimension(0) != weights->dimension(0), "Weights and dst OFMs should match");
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(dst->tensor_shape(), misc::shape_calculator::compute_conv3d_shape(src->tensor_shape(), weights->tensor_shape(), conv3d_info));
+ ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(src, dst);
+ }
+
+ return Status{};
+}
+} // namespace
+
+ClDirectConv3dKernel::ClDirectConv3dKernel()
+{
+ _type = CLKernelType::DIRECT;
+}
+
+void ClDirectConv3dKernel::configure(const CLCompileContext &compile_context, const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *biases, ITensorInfo *dst,
+ const Conv3dInfo &conv3d_info)
+{
+ ARM_COMPUTE_ERROR_ON_NULLPTR(src, weights, dst);
+
+ // Perform validation
+ ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(src, weights, biases, dst, conv3d_info));
+
+ // Create window and update padding
+ const DataType data_type = src->data_type();
+ const size_t src_width = src->dimension(1);
+ const size_t src_height = src->dimension(2);
+ const size_t src_depth = src->dimension(3);
+ const size_t src_channels = src->dimension(0);
+ const size_t dst_width = dst->dimension(1);
+ const size_t dst_height = dst->dimension(2);
+ const size_t dst_depth = dst->dimension(3);
+ const size_t dst_channels = dst->dimension(0);
+ const size_t weights_width = weights->dimension(2);
+ const size_t weights_height = weights->dimension(3);
+ const size_t weights_depth = weights->dimension(4);
+ const size_t pad_left = conv3d_info.padding.left;
+ const size_t pad_top = conv3d_info.padding.top;
+ const size_t pad_front = conv3d_info.padding.front;
+ const size_t conv_stride_x = conv3d_info.stride.x();
+ const size_t conv_stride_y = conv3d_info.stride.y();
+ const size_t conv_stride_z = conv3d_info.stride.z();
+
+ const size_t n0 = std::min(dst->dimension(0), static_cast<size_t>(4u));
+ const size_t m0 = (dst->tensor_shape()[0] > 16) ? ((data_type == DataType::F32) ? 2U : 4U) : 1U;
+ const size_t k0 = adjust_vec_size(8u, src->dimension(0));
+ const size_t partial_store_n0 = dst->dimension(0) % n0;
+
+ CLBuildOptions build_options;
+ build_options.add_option("-cl-fast-relaxed-math");
+ build_options.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(data_type));
+ build_options.add_option("-DACC_DATA_TYPE=float");
+ build_options.add_option("-DSRC_WIDTH=" + support::cpp11::to_string(src_width));
+ build_options.add_option("-DSRC_HEIGHT=" + support::cpp11::to_string(src_height));
+ build_options.add_option("-DSRC_DEPTH=" + support::cpp11::to_string(src_depth));
+ build_options.add_option("-DSRC_CHANNELS=" + support::cpp11::to_string(src_channels));
+ build_options.add_option("-DDST_WIDTH=" + support::cpp11::to_string(dst_width));
+ build_options.add_option("-DDST_HEIGHT=" + support::cpp11::to_string(dst_height));
+ build_options.add_option("-DDST_DEPTH=" + support::cpp11::to_string(dst_depth));
+ build_options.add_option("-DDST_CHANNELS=" + support::cpp11::to_string(dst_channels));
+ build_options.add_option("-DWEI_WIDTH=" + support::cpp11::to_string(weights_width));
+ build_options.add_option("-DWEI_HEIGHT=" + support::cpp11::to_string(weights_height));
+ build_options.add_option("-DWEI_DEPTH=" + support::cpp11::to_string(weights_depth));
+ build_options.add_option("-DSTRIDE_X=" + support::cpp11::to_string(conv_stride_x));
+ build_options.add_option("-DSTRIDE_Y=" + support::cpp11::to_string(conv_stride_y));
+ build_options.add_option("-DSTRIDE_Z=" + support::cpp11::to_string(conv_stride_z));
+ build_options.add_option("-DPAD_LEFT=" + support::cpp11::to_string(pad_left));
+ build_options.add_option("-DPAD_TOP=" + support::cpp11::to_string(pad_top));
+ build_options.add_option("-DPAD_FRONT=" + support::cpp11::to_string(pad_front));
+ build_options.add_option("-DN0=" + support::cpp11::to_string(n0));
+ build_options.add_option("-DM0=" + support::cpp11::to_string(m0));
+ build_options.add_option("-DK0=" + support::cpp11::to_string(k0));
+ build_options.add_option("-DPARTIAL_N0=" + support::cpp11::to_string(partial_store_n0));
+ build_options.add_option_if(biases != nullptr, std::string("-DHAS_BIAS"));
+
+ std::string kernel_name = "direct_convolution3d_ndhwc";
+ _kernel = create_kernel(compile_context, kernel_name, build_options.options());
+
+ // Configure kernel window
+ Window win = calculate_max_window(*dst, Steps(n0, m0));
+ ICLKernel::configure_internal(win);
+
+ // Set config_id for enabling LWS tuning
+ _config_id = kernel_name;
+ _config_id += "_";
+ _config_id += lower_string(string_from_data_type(data_type));
+ _config_id += "_";
+ _config_id += support::cpp11::to_string(weights_width);
+ _config_id += "_";
+ _config_id += support::cpp11::to_string(weights_height);
+ _config_id += "_";
+ _config_id += support::cpp11::to_string(weights_depth);
+ _config_id += "_";
+ _config_id += support::cpp11::to_string(conv_stride_x);
+ _config_id += "_";
+ _config_id += support::cpp11::to_string(conv_stride_y);
+ _config_id += "_";
+ _config_id += support::cpp11::to_string(conv_stride_z);
+ _config_id += "_";
+ _config_id += support::cpp11::to_string(dst_width);
+ _config_id += "_";
+ _config_id += support::cpp11::to_string(dst_height);
+ _config_id += "_";
+ _config_id += support::cpp11::to_string(dst_channels);
+}
+
+Status ClDirectConv3dKernel::validate(const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *dst, const Conv3dInfo &conv3d_info)
+{
+ ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(src, weights, biases, dst, conv3d_info));
+ return Status{};
+}
+
+void ClDirectConv3dKernel::run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue)
+{
+ ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
+ ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(IKernel::window(), window);
+
+ const auto src = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_0));
+ const auto weights = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_1));
+ const auto biases = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_2));
+ auto dst = utils::cast::polymorphic_downcast<ICLTensor *>(tensors.get_tensor(TensorType::ACL_DST));
+
+ // Get initial windows
+ Window slice = window.first_slice_window_3D();
+ slice.set(Window::DimY, Window::Dimension(0, ceil_to_multiple(dst->info()->dimension(1) * dst->info()->dimension(2) * dst->info()->dimension(3), slice.y().step()), slice.y().step()));
+ slice.set(Window::DimZ, Window::Dimension(0, dst->info()->dimension(4), 1));
+
+ unsigned int idx = 0;
+ add_4D_tensor_argument(idx, src, slice);
+ add_4D_tensor_argument(idx, dst, slice);
+ add_4D_tensor_argument(idx, weights, slice);
+ if(biases != nullptr)
+ {
+ add_1D_tensor_argument(idx, biases, slice);
+ }
+ enqueue(queue, *this, slice, lws_hint());
+}
+} // namespace kernels
+} // namespace opencl
+} // namespace arm_compute
diff --git a/src/gpu/cl/kernels/ClDirectConv3dKernel.h b/src/gpu/cl/kernels/ClDirectConv3dKernel.h
new file mode 100644
index 0000000..9ac8f0d
--- /dev/null
+++ b/src/gpu/cl/kernels/ClDirectConv3dKernel.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2021 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_CL_DIRECT_CONV3D_KERNEL_H
+#define ARM_COMPUTE_CL_DIRECT_CONV3D_KERNEL_H
+
+#include "src/gpu/cl/IClKernel.h"
+
+namespace arm_compute
+{
+class CLCompileContext;
+struct Conv3dInfo;
+
+namespace opencl
+{
+namespace kernels
+{
+/** Interface for the direct convolution 3d kernel. */
+class ClDirectConv3dKernel : public IClKernel
+{
+public:
+ /** Construtor */
+ ClDirectConv3dKernel();
+ /** Prevent instances of this class from being copied (As this class contains pointers) */
+ ClDirectConv3dKernel(const ClDirectConv3dKernel &) = delete;
+ /** Prevent instances of this class from being copied (As this class contains pointers) */
+ ClDirectConv3dKernel &operator=(const ClDirectConv3dKernel &) = delete;
+ /** Default move constructor */
+ ClDirectConv3dKernel(ClDirectConv3dKernel &&) = default;
+ /** Default move assignment operator */
+ ClDirectConv3dKernel &operator=(ClDirectConv3dKernel &&) = default;
+ /** Set the src, weights, biases and dst tensors info.
+ *
+ * Valid data layouts:
+ * - NDHWC
+ *
+ * Valid data type configurations:
+ * |src0 |src1 |src2 |dst |
+ * |:--------------|:--------------|:------|:--------------|
+ * |F16 |F16 |F16 |F16 |
+ * |F32 |F32 |F32 |F32 |
+ *
+ * @param[in] compile_context The compile context to be used.
+ * @param[in] src Source tensor. 4 lower dimensions represent a single src [IFM, width, height, depth],
+ * while every optional dimension from 5 and above represent a batch of srcs.
+ * @param[in] weights Weights tensor. Weights are 5D tensor with dimensions [OFM, IFM, kernel_w, kernel_h, kernel_d].
+ * @param[in] biases Biases tensor. Shared biases supported. Biases are 1D tensor with dimensions [OFM].
+ * @param[out] dst Destination tensor. 4 lower dimensions represent a single dst [OFM, width, height, depth], while the rest represent batch of dsts.
+ * @param[in] conv3d_info Contains strides, padding, rounding, activation, dilation and fast math information. Activation and fast math are currently unused.
+ */
+ void configure(const CLCompileContext &compile_context, const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *biases, ITensorInfo *dst, const Conv3dInfo &conv3d_info);
+ /** Static function to check if given info will lead to a valid configuration
+ *
+ * Similar to ClDirectConv3dKernel::configure()
+ *
+ * @return a status
+ */
+ static Status validate(const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *dst, const Conv3dInfo &conv3d_info);
+
+ // Inherited methods overridden:
+ void run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue) override;
+};
+} // namespace kernels
+} // namespace opencl
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_CL_DIRECT_CONV3D_KERNEL_H */