COMPMID-2053: Fuse bias addition with CLGEMMMatrixMultiplyReshapedKernel
Change-Id: I5bfd38c94a6fd18a1cba2104f7e1b04e7bef6ec2
Signed-off-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
Reviewed-on: https://review.mlplatform.org/c/1359
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
diff --git a/src/runtime/CL/functions/CLGEMM.cpp b/src/runtime/CL/functions/CLGEMM.cpp
index 21a9fce..94b318c 100644
--- a/src/runtime/CL/functions/CLGEMM.cpp
+++ b/src/runtime/CL/functions/CLGEMM.cpp
@@ -34,6 +34,7 @@
#include "arm_compute/core/Types.h"
#include "arm_compute/core/Utils.h"
#include "arm_compute/core/Validate.h"
+#include "arm_compute/core/utils/helpers/float_ops.h"
#include "arm_compute/core/utils/misc/ShapeCalculator.h"
#include "arm_compute/runtime/CL/CLScheduler.h"
#include "arm_compute/runtime/ITensorAllocator.h"
@@ -189,10 +190,6 @@
void CLGEMM::configure_reshaped_v2(const ICLTensor *a, const ICLTensor *b, const ICLTensor *c, ICLTensor *output, float alpha, float beta, const GEMMInfo &gemm_info)
{
- ARM_COMPUTE_ERROR_ON(c != nullptr);
- ARM_COMPUTE_UNUSED(beta);
- ARM_COMPUTE_UNUSED(c);
-
DataType data_type = a->info()->data_type();
bool reinterpret_input_as_3d = gemm_info.reinterpret_input_as_3d();
const unsigned int m = reinterpret_input_as_3d ? (a->info()->dimension(1) * a->info()->dimension(2)) : a->info()->dimension(1);
@@ -201,12 +198,13 @@
const unsigned int batch_size = reinterpret_input_as_3d ? a->info()->dimension(3) : a->info()->dimension(2);
const int depth_output_gemm3d = gemm_info.depth_output_gemm3d();
const GPUTarget gpu_target = CLScheduler::get().target();
+ bool broadcast_bias = gemm_info.broadcast_bias();
// Set the target for the kernels
_reshape_lhs_kernel.set_target(gpu_target);
_mm_kernel.set_target(gpu_target);
- GEMMReshapeInfo reshape_info(m, n, k, 1, 1, depth_output_gemm3d, false);
+ GEMMReshapeInfo reshape_info(m, n, k, 1, 1, depth_output_gemm3d, false, broadcast_bias);
// Manage intermediate buffers
_memory_group.manage(&_tmp_a);
@@ -230,7 +228,7 @@
_reshape_rhs_kernel.configure(b, &_tmp_b, rhs_info);
// Configure and tune matrix multiply kernel
- _mm_reshaped_kernel.configure(&_tmp_a, &_tmp_b, output, alpha, lhs_info, rhs_info, reshape_info);
+ _mm_reshaped_kernel.configure(&_tmp_a, &_tmp_b, c, output, alpha, beta, lhs_info, rhs_info, reshape_info);
// Allocate intermediate tensors
_tmp_a.allocator()->allocate();
@@ -395,9 +393,9 @@
const unsigned int k = a->dimension(0);
const unsigned int batch_size = reinterpret_input_as_3d ? a->dimension(3) : a->dimension(2);
const int depth_output_gemm3d = gemm_info.depth_output_gemm3d();
- const bool add_c = (beta != 0.f && c != nullptr);
+ const bool broadcast_bias = gemm_info.broadcast_bias();
- const GEMMReshapeInfo reshape_info = GEMMReshapeInfo(m, n, k, 1, 1, depth_output_gemm3d, false);
+ const GEMMReshapeInfo reshape_info = GEMMReshapeInfo(m, n, k, 1, 1, depth_output_gemm3d, false, broadcast_bias);
GEMMLHSMatrixInfo lhs_info;
GEMMRHSMatrixInfo rhs_info;
@@ -416,13 +414,8 @@
ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMReshapeRHSMatrixKernel::validate(b, &tmp_b_info, rhs_info));
// Validate matrix multiply
- ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixMultiplyReshapedKernel::validate(&tmp_a_info, &tmp_b_info, output, alpha, lhs_info, rhs_info, reshape_info));
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixMultiplyReshapedKernel::validate(&tmp_a_info, &tmp_b_info, c, output, alpha, beta, lhs_info, rhs_info, reshape_info));
- if(add_c)
- {
- // Validate matrix addition kernel
- ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixAdditionKernel::validate(c, output, beta));
- }
return Status{};
}
@@ -486,31 +479,32 @@
// Select GEMMType
_gemm_type = select_gemm_type(m, n, k, a->info()->data_type(), _reshape_b_only_on_first_run, gpu_target);
- const bool is_gemm_reshaped_only_rhs = _gemm_type == GEMMType::RESHAPED_ONLY_RHS;
- const bool add_c = (beta != 0.f && c != nullptr);
- const bool is_beta_one = std::abs(1.0f - beta) < 0.00001f;
- const bool fuse_add = (is_beta_one && (c != nullptr && c->info()->num_dimensions() == 1)) || is_gemm_reshaped_only_rhs;
+ const bool is_fuse_add_c_supported = (_gemm_type == GEMMType::RESHAPED_V2) || (_gemm_type == GEMMType::RESHAPED_ONLY_RHS);
+ const bool add_c = (!(helpers::float_ops::is_zero(beta)) && c != nullptr);
+ const bool fuse_add_c = add_c && is_fuse_add_c_supported;
+
+ const ICLTensor *c_to_use = fuse_add_c ? c : nullptr;
switch(_gemm_type)
{
case GEMMType::NATIVE:
{
- configure_native(a, b, (add_c && fuse_add) ? c : nullptr, output, alpha, beta, gemm_info);
+ configure_native(a, b, c_to_use, output, alpha, beta, gemm_info);
break;
}
case GEMMType::RESHAPED_V1:
{
- configure_reshaped_v1(a, b, (add_c && fuse_add) ? c : nullptr, output, alpha, beta, gemm_info);
+ configure_reshaped_v1(a, b, c_to_use, output, alpha, beta, gemm_info);
break;
}
case GEMMType::RESHAPED_V2:
{
- configure_reshaped_v2(a, b, (add_c && fuse_add) ? c : nullptr, output, alpha, beta, gemm_info);
+ configure_reshaped_v2(a, b, c_to_use, output, alpha, beta, gemm_info);
break;
}
case GEMMType::RESHAPED_ONLY_RHS:
{
- configure_reshaped_only_rhs(a, b, (add_c && fuse_add) ? c : nullptr, output, alpha, beta, gemm_info);
+ configure_reshaped_only_rhs(a, b, c_to_use, output, alpha, beta, gemm_info);
break;
}
default:
@@ -520,7 +514,7 @@
}
// Configure matrix addition kernel
- if(add_c && !fuse_add)
+ if(add_c && !fuse_add_c)
{
_ma_kernel.configure(c, output, beta);
_run_addition = true;
@@ -539,26 +533,32 @@
// Select GEMMType
GEMMType gemm_type = select_gemm_type(m, n, k, a->data_type(), gemm_info.reshape_b_only_on_first_run(), gpu_target);
+ const bool is_fuse_add_c_supported = (gemm_type == GEMMType::RESHAPED_V2) || (gemm_type == GEMMType::RESHAPED_ONLY_RHS);
+ const bool add_c = (!(helpers::float_ops::is_zero(beta)) && c != nullptr);
+ const bool fuse_add_c = add_c && is_fuse_add_c_supported;
+
+ const ITensorInfo *c_to_use = fuse_add_c ? c : nullptr;
+
switch(gemm_type)
{
case GEMMType::NATIVE:
{
- ARM_COMPUTE_RETURN_ON_ERROR(validate_native(a, b, c, output, alpha, beta, gemm_info));
+ ARM_COMPUTE_RETURN_ON_ERROR(validate_native(a, b, c_to_use, output, alpha, beta, gemm_info));
break;
}
case GEMMType::RESHAPED_V1:
{
- ARM_COMPUTE_RETURN_ON_ERROR(validate_reshaped_v1(a, b, c, output, alpha, beta, gemm_info));
+ ARM_COMPUTE_RETURN_ON_ERROR(validate_reshaped_v1(a, b, c_to_use, output, alpha, beta, gemm_info));
break;
}
case GEMMType::RESHAPED_V2:
{
- ARM_COMPUTE_RETURN_ON_ERROR(validate_reshaped_v2(a, b, c, output, alpha, beta, gemm_info));
+ ARM_COMPUTE_RETURN_ON_ERROR(validate_reshaped_v2(a, b, c_to_use, output, alpha, beta, gemm_info));
break;
}
case GEMMType::RESHAPED_ONLY_RHS:
{
- ARM_COMPUTE_RETURN_ON_ERROR(validate_reshaped_only_rhs(a, b, c, output, alpha, beta, gemm_info));
+ ARM_COMPUTE_RETURN_ON_ERROR(validate_reshaped_only_rhs(a, b, c_to_use, output, alpha, beta, gemm_info));
break;
}
default:
@@ -567,6 +567,12 @@
}
}
+ // Validate matrix addition kernel
+ if(add_c && !fuse_add_c)
+ {
+ ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixAdditionKernel::validate(c, output, beta));
+ }
+
return Status{};
}