COMPMID-417: Port DepthConcatenate to QS8/QS16 for NEON/CL.
Change-Id: I3dddae63043c7aa18d908a4fc8abacf3c64f98ca
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/80081
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>
Reviewed-by: Steven Niu <steven.niu@arm.com>
diff --git a/src/core/CL/CLHelpers.cpp b/src/core/CL/CLHelpers.cpp
index dd87e77..1073b39 100644
--- a/src/core/CL/CLHelpers.cpp
+++ b/src/core/CL/CLHelpers.cpp
@@ -100,6 +100,19 @@
}
}
+std::string get_underlying_cl_type_from_data_type(const DataType &dt)
+{
+ switch(dt)
+ {
+ case DataType::QS8:
+ return "char";
+ case DataType::QS16:
+ return "short";
+ default:
+ return get_cl_type_from_data_type(dt);
+ }
+}
+
const std::string &string_from_target(GPUTarget target)
{
static std::map<GPUTarget, const std::string> gpu_target_map =
diff --git a/src/core/CL/cl_kernels/concatenate.cl b/src/core/CL/cl_kernels/concatenate.cl
index 00f5189..a92ab5b 100644
--- a/src/core/CL/cl_kernels/concatenate.cl
+++ b/src/core/CL/cl_kernels/concatenate.cl
@@ -25,29 +25,35 @@
/** This kernel concatenates the input tensor into the output tensor along the third dimension
*
- * @param[in] src_ptr Pointer to the source tensor. Supported data types: F32
+ * @param[in] src_ptr Pointer to the source tensor. Supported data types: QS8, QS16, F16, F32
* @param[in] src_stride_x Stride of the source tensor in X dimension (in bytes)
* @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes)
* @param[in] src_stride_y Stride of the source tensor in Y dimension (in bytes)
* @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes)
+ * @param[in] src_stride_z Stride of the source tensor in Z dimension (in bytes)
+ * @param[in] src_step_z src_stride_z * number of elements along Z processed per workitem(in bytes)
* @param[in] src_offset_first_element_in_bytes The offset of the first element in the source tensor
- * @param[out] dst_ptr Pointer to the destination tensor. Supported data types: F32
+ * @param[out] dst_ptr Pointer to the destination tensor. Supported data types: same as @p src_ptr
* @param[in] dst_stride_x Stride of the destination tensor in X dimension (in bytes)
* @param[in] dst_step_x dst_stride_x * number of elements along X processed per workitem(in bytes)
* @param[in] dst_stride_y Stride of the destination tensor in Y dimension (in bytes)
* @param[in] dst_step_y dst_stride_y * number of elements along Y processed per workitem(in bytes)
+ * @param[in] dst_stride_z Stride of the source tensor in Z dimension (in bytes)
+ * @param[in] dst_step_z dst_stride_z * number of elements along Z processed per workitem(in bytes)
* @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination tensor
- * @param[in] offset The offset to the first valid element of the output tensor in bytes
+ * @param[in] offsets The offsets to the first valid element of the output tensor in bytes
*/
__kernel void concatenate_depth(
- IMAGE_DECLARATION(src),
- IMAGE_DECLARATION(dst),
- unsigned int offset)
+ TENSOR3D_DECLARATION(src),
+ TENSOR3D_DECLARATION(dst),
+ int3 offsets)
{
- Image src = CONVERT_TO_IMAGE_STRUCT(src);
- Image dst = CONVERT_TO_IMAGE_STRUCT(dst);
+ Tensor3D src = CONVERT_TO_TENSOR3D_STRUCT(src);
+ Tensor3D dst = CONVERT_TO_TENSOR3D_STRUCT(dst);
- float4 source_values = vload4(0, (__global float *)src.ptr);
+ VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
+ source_values = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)tensor3D_offset(&src, -offsets.x, -offsets.y, 0));
- vstore4(source_values, 0, (__global float *)(dst.ptr + offset));
+ VSTORE(VEC_SIZE)
+ (source_values, 0, (__global DATA_TYPE *)(dst.ptr + offsets.z));
}
diff --git a/src/core/CL/cl_kernels/helpers.h b/src/core/CL/cl_kernels/helpers.h
index 29a43f7..0b6d92d 100644
--- a/src/core/CL/cl_kernels/helpers.h
+++ b/src/core/CL/cl_kernels/helpers.h
@@ -30,6 +30,12 @@
#define CLAMP(x, min_val, max_val) min(max(x, min_val), max_val)
+#define VLOAD_STR(size) vload##size
+#define VLOAD(size) VLOAD_STR(size)
+
+#define VSTORE_STR(size) vstore##size
+#define VSTORE(size) VSTORE_STR(size)
+
#define VEC_DATA_TYPE_STR(type, size) type##size
#define VEC_DATA_TYPE(type, size) VEC_DATA_TYPE_STR(type, size)
diff --git a/src/core/CL/kernels/CLDepthConcatenateKernel.cpp b/src/core/CL/kernels/CLDepthConcatenateKernel.cpp
index 73f1ba1..6a699ae 100644
--- a/src/core/CL/kernels/CLDepthConcatenateKernel.cpp
+++ b/src/core/CL/kernels/CLDepthConcatenateKernel.cpp
@@ -35,6 +35,10 @@
#include "arm_compute/core/Validate.h"
#include "arm_compute/core/Window.h"
+#include "support/ToolchainSupport.h"
+
+#include <map>
+
using namespace arm_compute;
CLDepthConcatenateKernel::CLDepthConcatenateKernel()
@@ -49,12 +53,22 @@
void CLDepthConcatenateKernel::configure(const ICLTensor *input, unsigned int depth_offset, ICLTensor *output)
{
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32);
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::F32);
+ static std::map<int, std::pair<std::string, int>> configs_map =
+ {
+ { 1, { "uchar", 16 } },
+ { 2, { "ushort", 8 } },
+ { 4, { "uint", 4 } },
+ { 8, { "ulong", 2 } },
+ };
+
+ ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32);
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT_POSITION(input, output);
ARM_COMPUTE_ERROR_ON(input->info()->dimension(2) + depth_offset > output->info()->dimension(2));
ARM_COMPUTE_ERROR_ON(input->info()->dimension(0) > output->info()->dimension(0));
ARM_COMPUTE_ERROR_ON(input->info()->dimension(1) > output->info()->dimension(1));
ARM_COMPUTE_ERROR_ON_MISMATCHING_SHAPES(3, input, output);
+ ARM_COMPUTE_ERROR_ON(configs_map.find(input->info()->element_size()) == configs_map.end());
// The gaps between the two lowest dimensions of input and output need to be divisible by 2
// Otherwise it is not clear how the padding should be added onto the input tensor
@@ -64,33 +78,44 @@
_input = input;
_output = output;
+ // Add build options
+ auto config = configs_map.find(static_cast<int>(input->info()->element_size()));
+ std::set<std::string> build_opts;
+ build_opts.emplace(("-DDATA_TYPE=" + config->second.first));
+ build_opts.emplace(("-DVEC_SIZE=" + support::cpp11::to_string(config->second.second)));
+
// Create kernel
- _kernel = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel("concatenate_depth"));
+ _kernel = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel("concatenate_depth", build_opts));
// Configure kernel window
_left_right = (output->info()->dimension(0) - input->info()->dimension(0)) / 2;
_top_bottom = (output->info()->dimension(1) - input->info()->dimension(1)) / 2;
- const unsigned int offset_to_first_elements_in_bytes = depth_offset * output->info()->strides_in_bytes()[2] + _left_right * output->info()->strides_in_bytes()[0] + _top_bottom *
- output->info()->strides_in_bytes()[1];
+ const int offset_to_first_elements_in_bytes = depth_offset * output->info()->strides_in_bytes()[2];
- const unsigned int num_elems_processed_per_iteration = 4;
- const unsigned int num_elems_read_per_iteration = 4;
+ const unsigned int num_elems_processed_per_iteration = 16 / input->info()->element_size();
+ const unsigned int num_elems_read_per_iteration = 16 / input->info()->element_size();
const unsigned int num_rows_read_per_iteration = 1;
// The window needs to be based on input as we copy all the depths of input
- Window win = calculate_max_enlarged_window(*input->info(), Steps(num_elems_processed_per_iteration), border_size());
+ Window win = calculate_max_window(*output->info(), Steps(num_elems_processed_per_iteration));
+ win.set(Window::DimZ, Window::Dimension(0, input->info()->tensor_shape().z(), 1));
+ AccessWindowRectangle input_access(input->info(), -_left_right, -_top_bottom, num_elems_read_per_iteration, num_rows_read_per_iteration);
AccessWindowHorizontal output_access(output->info(), 0, num_elems_processed_per_iteration);
-
- update_window_and_padding(win,
- AccessWindowRectangle(input->info(), -_left_right, -_top_bottom, num_elems_read_per_iteration, num_rows_read_per_iteration),
- output_access);
-
+ update_window_and_padding(win, input_access, output_access);
output_access.set_valid_region(win, ValidRegion(Coordinates(0, 0), output->info()->tensor_shape()));
- unsigned int idx = 2 * num_arguments_per_2D_tensor(); // Skip the input and output parameters
- _kernel.setArg<unsigned int>(idx, offset_to_first_elements_in_bytes);
+ unsigned int idx = 2 * num_arguments_per_3D_tensor(); // Skip the input and output parameters
+ const cl_int3 offsets =
+ {
+ {
+ static_cast<cl_int>(_left_right),
+ static_cast<cl_int>(_top_bottom),
+ static_cast<cl_int>(offset_to_first_elements_in_bytes),
+ }
+ };
+ _kernel.setArg<cl_int3>(idx, offsets);
ICLKernel::configure(win);
}
@@ -100,14 +125,14 @@
ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window);
- Window slice = window.first_slice_window_2D();
+ Window slice = window.first_slice_window_3D();
do
{
unsigned int idx = 0;
- add_2D_tensor_argument(idx, _input, slice);
- add_2D_tensor_argument(idx, _output, slice);
+ add_3D_tensor_argument(idx, _input, slice);
+ add_3D_tensor_argument(idx, _output, slice);
enqueue(queue, *this, slice);
}
- while(window.slide_window_slice_2D(slice));
+ while(window.slide_window_slice_3D(slice));
}
diff --git a/src/core/CL/kernels/CLFillBorderKernel.cpp b/src/core/CL/kernels/CLFillBorderKernel.cpp
index 2c751a4..7667491 100644
--- a/src/core/CL/kernels/CLFillBorderKernel.cpp
+++ b/src/core/CL/kernels/CLFillBorderKernel.cpp
@@ -76,7 +76,7 @@
// Define select type required by replicate border > 1
const DataType dt = tensor->info()->data_type();
- std::string select_type = get_cl_type_from_data_type(dt);
+ std::string select_type = get_underlying_cl_type_from_data_type(dt);
if(is_data_type_float(dt))
{
select_type = (DataType::F32 == dt) ? "int" : "short";
@@ -84,7 +84,7 @@
// Define build options
std::set<std::string> build_opts;
- build_opts.emplace(("-DDATA_TYPE=" + get_cl_type_from_data_type(dt)));
+ build_opts.emplace(("-DDATA_TYPE=" + get_underlying_cl_type_from_data_type(dt)));
build_opts.emplace(("-DSELECT_TYPE=" + select_type));
build_opts.emplace(("-DBORDER_SIZE_TOP=" + support::cpp11::to_string(border_size.top)));
build_opts.emplace(("-DBORDER_SIZE_BOTTOM=" + support::cpp11::to_string(border_size.bottom)));
@@ -119,9 +119,14 @@
case DataType::U8:
set_constant_border<uint8_t>(idx, constant_border_value);
break;
+ case DataType::QS8:
+ case DataType::S8:
+ set_constant_border<int8_t>(idx, constant_border_value);
+ break;
case DataType::U16:
set_constant_border<uint16_t>(idx, constant_border_value);
break;
+ case DataType::QS16:
case DataType::S16:
set_constant_border<int16_t>(idx, constant_border_value);
break;
diff --git a/src/core/NEON/kernels/NEDepthConcatenateKernel.cpp b/src/core/NEON/kernels/NEDepthConcatenateKernel.cpp
index 902490e..d58e4e0 100644
--- a/src/core/NEON/kernels/NEDepthConcatenateKernel.cpp
+++ b/src/core/NEON/kernels/NEDepthConcatenateKernel.cpp
@@ -27,17 +27,76 @@
#include "arm_compute/core/Helpers.h"
#include "arm_compute/core/IAccessWindow.h"
#include "arm_compute/core/ITensor.h"
+#include "arm_compute/core/NEON/NEFixedPoint.h"
#include "arm_compute/core/TensorInfo.h"
#include "arm_compute/core/Utils.h"
#include "arm_compute/core/Validate.h"
#include "arm_compute/core/Window.h"
#include <arm_neon.h>
+#include <cstdint>
using namespace arm_compute;
+namespace
+{
+// Overloads of 128-bit vector loads
+uint8x16_t loadq(const uint8_t *ptr)
+{
+ return vld1q_u8(ptr);
+}
+uint16x8_t loadq(const uint16_t *ptr)
+{
+ return vld1q_u16(ptr);
+}
+uint32x4_t loadq(const uint32_t *ptr)
+{
+ return vld1q_u32(ptr);
+}
+// Overloads of 128-bit vector stores
+void storeq(uint8_t *ptr, uint8x16_t val)
+{
+ return vst1q_u8(ptr, val);
+}
+void storeq(uint16_t *ptr, uint16x8_t val)
+{
+ return vst1q_u16(ptr, val);
+}
+void storeq(uint32_t *ptr, uint32x4_t val)
+{
+ return vst1q_u32(ptr, val);
+}
+
+template <typename T>
+void depth_concat(const ITensor *in, ITensor *out, std::pair<int, int> start_xy, int depth_offset, const Window &window)
+{
+ const int start_x = start_xy.first;
+ const int start_y = start_xy.second;
+
+ // Offset input
+ const int input_offset_to_first_elements_in_bytes = in->info()->offset_first_element_in_bytes() - start_x * in->info()->strides_in_bytes()[0] - start_y * in->info()->strides_in_bytes()[1];
+ uint8_t *input_ptr = in->buffer() + input_offset_to_first_elements_in_bytes;
+
+ // Offset output
+ const unsigned int output_offset_to_first_elements_in_bytes = out->info()->offset_first_element_in_bytes() + depth_offset * out->info()->strides_in_bytes()[2];
+ uint8_t *output_ptr = out->buffer() + output_offset_to_first_elements_in_bytes;
+
+ Iterator input(in, window);
+ Iterator output(out, window);
+
+ execute_window_loop(window, [&](const Coordinates & id)
+ {
+ const auto in_ptr = reinterpret_cast<const T *>(input_ptr + input.offset());
+ const auto out_ptr = reinterpret_cast<T *>(output_ptr + output.offset());
+
+ storeq(out_ptr, loadq(in_ptr));
+ },
+ input, output);
+}
+} // namespace
+
NEDepthConcatenateKernel::NEDepthConcatenateKernel()
- : _input(nullptr), _output(nullptr), _top_bottom(0), _left_right(0), _depth_offset(0)
+ : _func(nullptr), _input(nullptr), _output(nullptr), _top_bottom(0), _left_right(0), _depth_offset(0)
{
}
@@ -48,8 +107,9 @@
void NEDepthConcatenateKernel::configure(const ITensor *input, unsigned int depth_offset, ITensor *output)
{
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32);
- ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::F32);
+ ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32);
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
+ ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT_POSITION(input, output);
ARM_COMPUTE_ERROR_ON(input->info()->dimension(2) + depth_offset > output->info()->dimension(2));
ARM_COMPUTE_ERROR_ON(input->info()->dimension(0) > output->info()->dimension(0));
ARM_COMPUTE_ERROR_ON(input->info()->dimension(1) > output->info()->dimension(1));
@@ -60,18 +120,36 @@
ARM_COMPUTE_ERROR_ON((output->info()->dimension(0) - input->info()->dimension(0)) % 2);
ARM_COMPUTE_ERROR_ON((output->info()->dimension(1) - input->info()->dimension(1)) % 2);
+ _func = nullptr;
_input = input;
_output = output;
_depth_offset = depth_offset;
_left_right = (output->info()->dimension(0) - input->info()->dimension(0)) / 2;
_top_bottom = (output->info()->dimension(1) - input->info()->dimension(1)) / 2;
- const unsigned int num_elems_processed_per_iteration = 4;
- const unsigned int num_elems_read_per_iteration = 4;
+ switch(input->info()->data_type())
+ {
+ case DataType::QS8:
+ _func = &depth_concat<uint8_t>;
+ break;
+ case DataType::QS16:
+ case DataType::F16:
+ _func = &depth_concat<uint16_t>;
+ break;
+ case DataType::F32:
+ _func = &depth_concat<uint32_t>;
+ break;
+ default:
+ ARM_COMPUTE_ERROR("Unsupported data type.");
+ }
+
+ const unsigned int num_elems_processed_per_iteration = 16 / input->info()->element_size();
+ const unsigned int num_elems_read_per_iteration = 16 / input->info()->element_size();
const unsigned int num_rows_read_per_iteration = 1;
// The window needs to be based on input as we copy all the depths of input
- Window win = calculate_max_enlarged_window(*input->info(), Steps(num_elems_processed_per_iteration), border_size());
+ Window win = calculate_max_window(*output->info(), Steps(num_elems_processed_per_iteration));
+ win.set(Window::DimZ, Window::Dimension(0, input->info()->tensor_shape().z(), 1));
AccessWindowRectangle input_access(input->info(), -_left_right, -_top_bottom, num_elems_read_per_iteration, num_rows_read_per_iteration);
AccessWindowHorizontal output_access(output->info(), 0, num_elems_processed_per_iteration);
@@ -85,21 +163,7 @@
{
ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window);
+ ARM_COMPUTE_ERROR_ON(_func == nullptr);
- // Offset output
- const unsigned int offset_to_first_elements_in_bytes = _output->info()->offset_first_element_in_bytes() + _left_right * _output->info()->strides_in_bytes()[0] + _top_bottom *
- _output->info()->strides_in_bytes()[1] + _depth_offset * _output->info()->strides_in_bytes()[2];
- uint8_t *output_ptr = _output->buffer() + offset_to_first_elements_in_bytes;
-
- Iterator input(_input, window);
- Iterator output(_output, window);
-
- execute_window_loop(window, [&](const Coordinates & id)
- {
- const auto in_ptr = reinterpret_cast<const float *>(input.ptr());
- const auto out_ptr = reinterpret_cast<float *>(output_ptr + output.offset());
-
- vst1q_f32(out_ptr, vld1q_f32(in_ptr));
- },
- input, output);
+ (*_func)(_input, _output, std::make_pair(_left_right, _top_bottom), _depth_offset, window);
}