Rework gemm_mm_reshaped_only_rhs_ kernels with new macros
- Rework gemm_reshaped_rhs_only with new TILE macros
- Fuse post ops in gemm_reshaped_rhs_only
Resolves COMPMID-4890
Change-Id: I944948ecec6d08deaf3545b80cd3eeac26e44205
Signed-off-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/6944
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Sheri Zhang <sheri.zhang@arm.com>
diff --git a/src/gpu/cl/ClKernelLibrary.cpp b/src/gpu/cl/ClKernelLibrary.cpp
index 92a9d9c..856d377 100644
--- a/src/gpu/cl/ClKernelLibrary.cpp
+++ b/src/gpu/cl/ClKernelLibrary.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-2021 Arm Limited.
+ * Copyright (c) 2016-2022 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -281,14 +281,10 @@
{ "gemm_mm_reshaped_lhs_nt_rhs_t_texture_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.cl" },
{ "gemm_mm_reshaped_lhs_t_rhs_nt_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.cl" },
{ "gemm_mm_reshaped_lhs_t_rhs_nt_texture_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.cl" },
- { "gemm_mm_reshaped_only_rhs_nt", "common/gemm.cl" },
- { "gemm_mm_reshaped_only_rhs_nt_texture", "common/gemm.cl" },
- { "gemm_mm_reshaped_only_rhs_t", "common/gemm.cl" },
- { "gemm_mm_reshaped_only_rhs_t_texture", "common/gemm.cl" },
- { "gemm_mm_reshaped_only_rhs_nt_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl" },
- { "gemm_mm_reshaped_only_rhs_nt_texture_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl" },
- { "gemm_mm_reshaped_only_rhs_t_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl" },
- { "gemm_mm_reshaped_only_rhs_t_texture_post_act_eltwise_op_act", "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl" },
+ { "gemm_mm_reshaped_only_rhs_nt", "common/gemm_reshaped_rhs_only.cl" },
+ { "gemm_mm_reshaped_only_rhs_nt_texture", "common/gemm_reshaped_rhs_only.cl" },
+ { "gemm_mm_reshaped_only_rhs_t", "common/gemm_reshaped_rhs_only.cl" },
+ { "gemm_mm_reshaped_only_rhs_t_texture", "common/gemm_reshaped_rhs_only.cl" },
{ "gemm_lc_vm_f32", "common/gemm.cl" },
{ "gemm_reshape_lhs_matrix_nt", "common/gemm_utils.cl" },
{ "gemm_reshape_lhs_matrix_t", "common/gemm_utils.cl" },
@@ -590,6 +586,10 @@
#include "./cl_kernels/common/gemm_utils.clembed"
},
{
+ "common/gemm_reshaped_rhs_only.cl",
+#include "./cl_kernels/common/gemm_reshaped_rhs_only.clembed"
+ },
+ {
"common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_native.cl",
#include "./cl_kernels/common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_native.clembed"
},
@@ -598,10 +598,6 @@
#include "./cl_kernels/common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped.clembed"
},
{
- "common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.cl",
-#include "./cl_kernels/common/experimental/gemm_fused_post_ops/act_eltwise_op_act/gemm_mm_reshaped_only_rhs.clembed"
- },
- {
"common/gemmlowp.cl",
#include "./cl_kernels/common/gemmlowp.clembed"
},
diff --git a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp
index 29f9180..546a61e 100644
--- a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp
+++ b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2022 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -83,6 +83,7 @@
&& (!gemm_info.broadcast_bias),
"Bias addition only supported with broadcast mode in case the input or dst has to be reinterpreted as 3D");
ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.fp_mixed_precision, "Mixed precision not supported");
+ ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.has_pad_y, "Tensors cannot have padding along the Y direction");
ARM_COMPUTE_RETURN_ON_ERROR(gemm::validate_image2d_support_on_rhs(*src1, rhs_info));
ARM_COMPUTE_RETURN_ERROR_ON_MSG(!post_op_utils.is_post_op_sequence_supported(gemm_info.post_ops), "The sequence of Post Ops is not supported");
@@ -142,17 +143,8 @@
ARM_COMPUTE_UNUSED(src0, src1, src2);
unsigned int &num_elems_processed_per_iteration_x = num_elements_processed[0];
unsigned int &num_elems_processed_per_iteration_y = num_elements_processed[1];
- bool reinterpret_input_as_3d = gemm_info.reinterpret_input_as_3d;
bool reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0;
- // In case both input and dst have to be reinterpreted as 3D tensors,
- // force reinterpret_input_as_3d and reinterpret_output_as_3d to be false.
- // This approach should only be used when the input/dst tensors have pad on the y direction
- if((reinterpret_input_as_3d == reinterpret_output_as_3d) && gemm_info.has_pad_y)
- {
- reinterpret_output_as_3d = false;
- }
-
TensorInfo tmp_info(*dst);
if(reinterpret_output_as_3d)
@@ -200,19 +192,10 @@
_use_dummy_work_items = preferred_dummy_work_items_support(CLKernelLibrary::get().get_device());
_add_bias = src2 != nullptr;
_export_to_cl_image = rhs_info.export_to_cl_image;
- _has_pad_y = gemm_info.has_pad_y;
_num_post_op_args = gemm_info.post_ops.total_num_arguments();
auto padding_info = get_padding_info({ src0, src1, src2, dst });
- // In case both input and dst have to be reinterpreted as 3D tensors,
- // force reinterpret_input_as_3d and reinterpret_output_as_3d to be false.
- if((_reinterpret_input_as_3d == _reinterpret_output_as_3d) && _has_pad_y)
- {
- _reinterpret_input_as_3d = false;
- _reinterpret_output_as_3d = false;
- }
-
// Check if we need to slide the matrix B
const unsigned int num_dimensions_src0 = src0->num_dimensions();
_slide_matrix_b = (src1->num_dimensions() >= num_dimensions_src0);
@@ -229,10 +212,6 @@
// This means that the actual m used by the kernel is given by dst->dimension(1) and not by gemm_info.m
const unsigned int internal_m = _reinterpret_output_as_3d ? gemm_info.m : dst->dimension(1);
- // These variables are used only if gemm_info.has_pad_y == true
- const unsigned int h_gemm_3d = _reinterpret_output_as_3d ? dst->dimension(1) : src0->dimension(1);
- const unsigned int d_gemm_3d = _reinterpret_output_as_3d ? dst->dimension(2) : src0->dimension(2);
-
// Shrink M0 to be always <= M (internal_m) to prevent out-of-bounds reads.
// NOTE: This might have implications on heuristics and performance
const unsigned int internal_m0 = std::min(internal_m, lhs_info.m0);
@@ -243,31 +222,26 @@
_m = internal_m;
_n = gemm_info.n;
_k = gemm_info.k;
+
// Create build options
CLBuildOptions build_opts;
build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(src0->data_type()));
- build_opts.add_option_if(!(helpers::float_ops::is_one(alpha)), "-DALPHA=" + float_to_string_with_full_precision(alpha));
- build_opts.add_option_if(src2 != nullptr, "-DBETA=" + float_to_string_with_full_precision(beta));
- build_opts.add_option_if(helpers::float_ops::is_one(beta), "-DUNIT_BETA");
- build_opts.add_option_if(gemm_info.broadcast_bias, "-DBROADCAST_BIAS");
- build_opts.add_option_if(!_slide_matrix_b, "-DMATRIX_B_DEPTH=" + support::cpp11::to_string(src1->dimension(2)));
- build_opts.add_option_if(rhs_info.interleave, "-DRHS_INTERLEAVE");
- build_opts.add_option_if(_use_dummy_work_items, "-DDUMMY_WORK_ITEMS");
- build_opts.add_option_if(rhs_info.export_to_cl_image, "-DOPENCL_IMAGE_SUPPORT");
- build_opts.add_option("-DRHS_HEIGHT=" + support::cpp11::to_string(src1->dimension(1)));
build_opts.add_option("-DM0=" + support::cpp11::to_string(internal_m0));
build_opts.add_option("-DN0=" + support::cpp11::to_string(rhs_info.n0));
build_opts.add_option("-DK0=" + support::cpp11::to_string(rhs_info.k0));
build_opts.add_option("-DH0=" + support::cpp11::to_string(rhs_info.h0));
build_opts.add_option("-DPARTIAL_STORE_M0=" + support::cpp11::to_string(partial_store_m0));
build_opts.add_option("-DPARTIAL_STORE_N0=" + support::cpp11::to_string(partial_store_n0));
- if(_has_pad_y)
- {
- build_opts.add_option_if(_reinterpret_input_as_3d, "-DREINTERPRET_INPUT_AS_3D");
- build_opts.add_option_if(_reinterpret_output_as_3d, "-DREINTERPRET_OUTPUT_AS_3D");
- build_opts.add_option_if(_reinterpret_input_as_3d || _reinterpret_output_as_3d, "-DHEIGHT_GEMM3D=" + support::cpp11::to_string(h_gemm_3d));
- build_opts.add_option_if(_reinterpret_input_as_3d || _reinterpret_output_as_3d, "-DDEPTH_GEMM3D=" + support::cpp11::to_string(d_gemm_3d));
- }
+ build_opts.add_option_if(_use_dummy_work_items, "-DDUMMY_WORK_ITEMS");
+ build_opts.add_option_if(!(helpers::float_ops::is_one(alpha)), "-DALPHA=" + float_to_string_with_full_precision(alpha));
+ build_opts.add_option_if(src2 != nullptr, "-DBETA=" + float_to_string_with_full_precision(beta));
+ build_opts.add_option_if(helpers::float_ops::is_one(beta), "-DUNIT_BETA");
+ build_opts.add_option_if(gemm_info.broadcast_bias, "-DBROADCAST_BIAS");
+ build_opts.add_option_if(!_slide_matrix_b, "-DMATRIX_B_DEPTH=" + support::cpp11::to_string(src1->dimension(2)));
+ build_opts.add_option_if(rhs_info.interleave, "-DRHS_INTERLEAVE");
+ build_opts.add_option_if(gemm_info.k % rhs_info.k0, "-DRUN_LEFTOVER_K0");
+ build_opts.add_option_if((gemm_info.k % rhs_info.k0) && rhs_info.transpose, "-DPARTIAL_K=" + support::cpp11::to_string(gemm_info.k % rhs_info.k0));
+
// If post_ops are used, then we disable the use of gemm_info.activation_info
if(gemm_info.post_ops.size() > 0)
{
@@ -275,15 +249,15 @@
}
else
{
- build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DACTIVATION_TYPE=" + lower_string(string_from_activation_func(gemm_info.activation_info.activation())));
- build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DA_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.a()));
- build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DB_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.b()));
+ build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DPOST_OP1");
+ build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DP1_ACTIVATION_TYPE=" + lower_string(string_from_activation_func(gemm_info.activation_info.activation())));
+ build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DP1_ACTIVATION_A_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.a()));
+ build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DP1_ACTIVATION_B_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.b()));
}
std::string kernel_name("gemm_mm_reshaped_only_rhs_");
kernel_name += rhs_info.transpose ? "t" : "nt";
- kernel_name += rhs_info.export_to_cl_image ? "_texture" : "";
- post_op_utils.set_post_ops_cl_kernel_name(kernel_name, gemm_info.post_ops);
+ kernel_name += _export_to_cl_image ? "_texture" : "";
// A macro guard to compile ONLY the kernel of interest
build_opts.add_option("-D" + upper_string(kernel_name));
@@ -294,11 +268,8 @@
// Set config_id for enabling LWS tuning
_config_id = kernel_name;
_config_id += "_";
- _config_id += (_has_pad_y ? "" : "no_pad_y_");
_config_id += (_add_bias ? "add_bias_" : "");
_config_id += (gemm_info.broadcast_bias ? "broadcast_bias_" : "");
- _config_id += (_reinterpret_input_as_3d ? "3di_" : "");
- _config_id += (_reinterpret_output_as_3d ? "3do_" : "");
_config_id += (gemm_info.activation_info.enabled() ? "fused_activation_" : "");
_config_id += lower_string(string_from_data_type(src0->data_type()));
_config_id += "_";
@@ -350,24 +321,12 @@
ARM_COMPUTE_ERROR_ON(src1->info()->strides_in_bytes()[3] != 0);
}
- const size_t lhs_idx_batch_size = _reinterpret_input_as_3d && !_has_pad_y ? 3u : 2u;
- const size_t rhs_idx_batch_size = 2u;
- const size_t bia_idx_batch_size = 2u;
- const size_t out_idx_batch_size = _reinterpret_output_as_3d && !_has_pad_y ? 3u : 2u;
-
Window slice = window.first_slice_window_3D();
Window slice_matrix_b = slice;
slice_matrix_b.set(Window::DimX, Window::Dimension(0, 1, 1));
slice_matrix_b.set(Window::DimY, Window::Dimension(0, 1, 1));
- // Get cross plane pads
- const unsigned int total_cross_plane_pad_lhs = src0->info()->padding().top + src0->info()->padding().bottom;
- const unsigned int total_cross_plane_pad_out = dst->info()->padding().top + dst->info()->padding().bottom;
-
- // The execution should fail if we try to run with has_pad_y = false but we have padding in either the LHS or DST tensor
- ARM_COMPUTE_ERROR_ON(!_has_pad_y && ((total_cross_plane_pad_lhs != 0) || (total_cross_plane_pad_out != 0)));
-
cl::Image2D src1_image2d;
if(_export_to_cl_image)
@@ -391,63 +350,30 @@
unsigned int idx = 0;
// LHS buffer
- add_2D_tensor_argument(idx, src0, slice);
+ add_3d_tensor_nhw_argument(idx, src0);
// RHS buffer or RHS OpenCL image (_export_to_cl_image == true)
if(_export_to_cl_image)
{
_kernel.setArg(idx++, src1_image2d);
}
- else
- {
- add_2D_tensor_argument(idx, src1, slice_b);
- }
+ add_3d_tensor_nhw_argument(idx, src1);
// Bias buffer (_add_bias == true)
- add_2D_tensor_argument_if(_add_bias, idx, src2, slice);
-
- // dst buffer
- add_2D_tensor_argument(idx, dst, slice);
+ if(_add_bias)
+ {
+ add_3d_tensor_nhw_argument(idx, src2);
+ }
// post op argument buffers
for(size_t i = 0; i < _num_post_op_args; ++i)
{
const auto post_op_arg = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(experimental::get_post_op_arg_type(i)));
- add_2D_tensor_argument(idx, post_op_arg, slice);
+ add_3d_tensor_nhw_argument(idx, post_op_arg);
}
- // LHS stride_z
- _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(src0->info()->strides_in_bytes()[lhs_idx_batch_size]));
-
- // RHS stride_z (not used if _export_to_cl_image == true)
- _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(src1->info()->strides_in_bytes()[rhs_idx_batch_size]));
-
- // Bias stride_z (if _add_bias == true)
- if(_add_bias)
- {
- _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(src2->info()->strides_in_bytes()[bia_idx_batch_size]));
- }
-
- // dst stride_z
- _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(dst->info()->strides_in_bytes()[out_idx_batch_size]));
- // post op argument stride_z
- for(size_t i = 0; i < _num_post_op_args; ++i)
- {
- const auto post_op_arg = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(experimental::get_post_op_arg_type(i)));
- _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(post_op_arg->info()->strides_in_bytes()[2]));
- }
-
- // Cross-plan padding (if _reinterpret_input_as_3d = true)
- if(_reinterpret_input_as_3d && _has_pad_y)
- {
- _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(total_cross_plane_pad_lhs));
- }
-
- // Cross-plan padding (if reinterpret_output_as_3d = true)
- if(_reinterpret_output_as_3d && _has_pad_y)
- {
- _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(total_cross_plane_pad_out));
- }
+ // dst buffer
+ add_3d_tensor_nhw_argument(idx, dst);
// Pass m, n and k at runtime as signed ints, to ensure results of any subractions they could be operand in, would still be signed.
_kernel.setArg<cl_int>(idx++, _m);
diff --git a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.h b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.h
index ec5878d..297e681 100644
--- a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.h
+++ b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2021 Arm Limited.
+ * Copyright (c) 2019-2022 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -96,7 +96,6 @@
bool _use_dummy_work_items{ false };
bool _add_bias{ false };
bool _export_to_cl_image{ false };
- bool _has_pad_y{ false };
signed int _m{ 1 };
signed int _n{ 1 };
signed int _k{ 1 };
diff --git a/src/gpu/cl/operators/ClGemm.cpp b/src/gpu/cl/operators/ClGemm.cpp
index 5557385..766aec3 100644
--- a/src/gpu/cl/operators/ClGemm.cpp
+++ b/src/gpu/cl/operators/ClGemm.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2021 Arm Limited.
+ * Copyright (c) 2017-2022 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -452,9 +452,6 @@
kernel_info.has_pad_y = false;
ARM_COMPUTE_RETURN_ON_ERROR(ClGemmMatrixMultiplyReshapedOnlyRhsKernel::validate(a, &tmp_b_info, c, output, alpha, beta, lhs_info, rhs_info, kernel_info));
- kernel_info.has_pad_y = true;
- ARM_COMPUTE_RETURN_ON_ERROR(ClGemmMatrixMultiplyReshapedOnlyRhsKernel::validate(a, &tmp_b_info, c, output, alpha, beta, lhs_info, rhs_info, kernel_info));
-
return Status{};
}