COMPMID-1979: Fuse Activation Function in CLGEMM - part 4
Fused activation function in CLGEMM
Change-Id: I644fdf09349325c0b3a2cd5fef2a3ea2c974149d
Signed-off-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
Reviewed-on: https://review.mlplatform.org/c/1640
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
diff --git a/src/runtime/CL/functions/CLGEMMConvolutionLayer.cpp b/src/runtime/CL/functions/CLGEMMConvolutionLayer.cpp
index 99f045a..be6be04 100644
--- a/src/runtime/CL/functions/CLGEMMConvolutionLayer.cpp
+++ b/src/runtime/CL/functions/CLGEMMConvolutionLayer.cpp
@@ -91,22 +91,27 @@
}
CLGEMMConvolutionLayer::CLGEMMConvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager)
- : _memory_group(memory_manager), _reshape_weights(), _im2col_kernel(), _mm_gemm(memory_manager), _mm_gemmlowp(memory_manager), _col2im_kernel(), _activationlayer_function(), _add_bias_kernel(),
- _original_weights(nullptr), _im2col_output(), _weights_reshaped(), _gemm_output(), _data_layout(DataLayout::NCHW), _append_bias(false), _skip_im2col(false), _skip_col2im(false), _is_quantized(false),
- _is_activationlayer_enabled(false), _is_prepared(false), _run_addition(true)
+ : _memory_group(memory_manager), _reshape_weights(), _im2col_kernel(), _mm_gemm(memory_manager), _mm_gemmlowp(memory_manager), _col2im_kernel(), _activationlayer_function(),
+ _original_weights(nullptr), _im2col_output(), _weights_reshaped(), _gemm_output(), _skip_im2col(false), _skip_col2im(false), _is_quantized(false), _fuse_activation(true), _is_prepared(false)
{
}
void CLGEMMConvolutionLayer::configure_mm(const ICLTensor *input, const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, const GEMMLowpOutputStageInfo &gemmlowp_output_stage,
- int gemm_3d_depth)
+ int gemm_3d_depth, const ActivationLayerInfo &act_info)
{
ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights);
- ARM_COMPUTE_ERROR_THROW_ON(validate_mm(input->info(), weights->info(), biases != nullptr ? biases->info() : nullptr, output->info(), gemmlowp_output_stage, gemm_3d_depth, _skip_im2col,
- _run_addition));
+ ARM_COMPUTE_ERROR_THROW_ON(validate_mm(input->info(), weights->info(), biases != nullptr ? biases->info() : nullptr, output->info(), gemmlowp_output_stage, gemm_3d_depth, _skip_im2col, act_info));
- const GEMMInfo &gemm_info = GEMMInfo(false, false, true /* Reshape weights only for the first run */,
- gemm_3d_depth, _skip_im2col /* Reinterpret the input as 3D if im2col is skipped */,
- false, gemmlowp_output_stage);
+ const GEMMInfo &gemm_info = GEMMInfo(false, // is_a_reshaped
+ false, // is_b_reshaped
+ true, // reshape_b_only_on_first_run
+ gemm_3d_depth, // depth_output_gemm3d
+ _skip_im2col, // reinterpret_input_as_3d
+ false, // retain_internal_weights
+ gemmlowp_output_stage, // gemmlowp_output_stage
+ false, // fp_mixed_precision
+ true, // broadcast_bias
+ act_info); // activation_info
if(_is_quantized)
{
@@ -126,21 +131,26 @@
}
else
{
- // Bias does not need to be added in GEMM if im2col is being used or the Matrix Addition kernel needs to be run
- const bool skip_bias_in_gemm = _run_addition || !_skip_im2col;
// Configure matrix multiply function
- _mm_gemm.configure(input, weights, (skip_bias_in_gemm) ? nullptr : biases, output, 1.0f, 1.0f, gemm_info);
+ _mm_gemm.configure(input, weights, biases, output, 1.0f, 1.0f, gemm_info);
}
}
Status CLGEMMConvolutionLayer::validate_mm(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output,
- const GEMMLowpOutputStageInfo &gemmlowp_output_stage, int gemm_3d_depth, bool skip_im2col, bool run_addition)
+ const GEMMLowpOutputStageInfo &gemmlowp_output_stage, int gemm_3d_depth, bool skip_im2col, const ActivationLayerInfo &act_info)
{
const bool is_quantized = is_data_type_quantized_asymmetric(input->data_type());
- const GEMMInfo &gemm_info = GEMMInfo(false, false, true /* Reshape weights only for the first run */,
- gemm_3d_depth, skip_im2col /* Reinterpret the input as 3D if im2col is skipped */,
- false, gemmlowp_output_stage);
+ const GEMMInfo &gemm_info = GEMMInfo(false, // is_a_reshaped
+ false, // is_b_reshaped
+ true, // reshape_b_only_on_first_run
+ gemm_3d_depth, // depth_output_gemm3d
+ skip_im2col, // reinterpret_input_as_3d
+ false, // retain_internal_weights
+ gemmlowp_output_stage, // gemmlowp_output_stage
+ false, // fp_mixed_precision
+ true, // broadcast_bias
+ act_info); // activation_info
if(is_quantized)
{
@@ -159,10 +169,8 @@
}
else
{
- // Bias does not need to be added in GEMM if im2col is being used or the Matrix Addition kernel needs to be run
- const bool skip_bias_in_gemm = run_addition || !skip_im2col;
// Perform validation step on Matrix multiply function
- return CLGEMM::validate(input, weights, (skip_bias_in_gemm) ? nullptr : biases, output, 1.0f, 1.0f, gemm_info);
+ return CLGEMM::validate(input, weights, biases, output, 1.0f, 1.0f, gemm_info);
}
}
@@ -194,15 +202,14 @@
const UniformQuantizationInfo wq_info = weights->info()->quantization_info().uniform();
const UniformQuantizationInfo oq_info = output->info()->quantization_info().uniform();
- _is_prepared = weights_info.retain_internal_weights();
- _original_weights = weights;
- _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type());
- _data_layout = data_layout;
- _skip_im2col = (data_layout == DataLayout::NHWC && kernel_width == 1 && kernel_height == 1 && conv_info.stride().first == 1 && conv_info.stride().second == 1);
- _skip_col2im = data_layout == DataLayout::NHWC;
- _append_bias = (biases != nullptr) && (!_is_quantized);
- _is_activationlayer_enabled = act_info.enabled();
- _run_addition = (_skip_im2col) && (_append_bias);
+ _is_prepared = weights_info.retain_internal_weights();
+ _original_weights = weights;
+ _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type());
+ _skip_im2col = (data_layout == DataLayout::NHWC && kernel_width == 1 && kernel_height == 1 && conv_info.stride().first == 1 && conv_info.stride().second == 1);
+ _skip_col2im = data_layout == DataLayout::NHWC;
+
+ // Only for quantize there are few cases where we cannot fuse the activation function in GEMM
+ _fuse_activation = true;
// Set the GPU target for im2col and col2im
_im2col_kernel.set_target(CLScheduler::get().target());
@@ -211,8 +218,6 @@
const ICLTensor *gemm_input_to_use = input;
ICLTensor *gemm_output_to_use = output;
- const ICLTensor *biases_to_use = (_append_bias && !_skip_im2col) ? biases : nullptr;
-
// Get parameters from conv_info
unsigned int stride_x = 0;
unsigned int stride_y = 0;
@@ -230,9 +235,22 @@
unsigned int mat_weights_cols = weights->info()->dimension(idx_kernels) / num_groups;
- // _weights_reshaped will be auto configured in the kernel.
- // Just append biases and do not transpose 1xW as it will be reshaped in CLGEMM
- _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped, num_groups);
+ const ICLTensor *biases_to_use = biases;
+ bool append_bias = false;
+
+ if(num_groups != 1 && biases != nullptr)
+ {
+ // num_groups != 1 can only be for NCHW
+ // Since it is missing an utility function to reshape the biases, we append the biases into the weights tensor
+ biases_to_use = nullptr;
+ append_bias = true;
+
+ _reshape_weights.configure(weights, biases, &_weights_reshaped, num_groups);
+ }
+ else
+ {
+ _reshape_weights.configure(weights, nullptr, &_weights_reshaped, num_groups);
+ }
// Create tensor to store im2col reshaped inputs
if(!_skip_im2col)
@@ -240,7 +258,7 @@
_memory_group.manage(&_im2col_output);
// Configure and tune im2col. im2col output shape is auto-initialized
- _im2col_kernel.configure(input, &_im2col_output, Size2D(kernel_width, kernel_height), conv_info, _append_bias, dilation, num_groups);
+ _im2col_kernel.configure(input, &_im2col_output, Size2D(kernel_width, kernel_height), conv_info, append_bias, dilation, num_groups);
// Set quantization info
_im2col_output.info()->set_quantization_info(input->info()->quantization_info());
@@ -249,11 +267,6 @@
// Update GEMM input
gemm_input_to_use = &_im2col_output;
}
- else if(_append_bias)
- {
- // Configure add bias kernel
- _add_bias_kernel.configure(ArithmeticOperation::ADD, output, biases, output, ConvertPolicy::SATURATE);
- }
// Create GEMM output tensor
if(!_skip_col2im)
@@ -299,16 +312,20 @@
ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU
};
- if(_is_activationlayer_enabled && supported_acts.count(act_info.activation()) != 0)
+ if(act_info.enabled())
{
- const int a_const_int = quantize_qasymm8(act_info.a(), output_quant_info);
- const int b_const_int = quantize_qasymm8(act_info.b(), output_quant_info);
+ if(supported_acts.count(act_info.activation()) != 0)
+ {
+ const int a_const_int = quantize_qasymm8(act_info.a(), output_quant_info);
+ const int b_const_int = quantize_qasymm8(act_info.b(), output_quant_info);
- min_activation = act_info.activation() != ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU ? output_quant_info.offset : b_const_int;
- max_activation = act_info.activation() == ActivationLayerInfo::ActivationFunction::RELU ? 255 : a_const_int;
-
- // If the activation layer is RELU, BOUNDED_RELU or LU_BOUNDED_RELU, we can use the GEMMLowp output stage to perform this operation
- _is_activationlayer_enabled = false;
+ min_activation = act_info.activation() != ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU ? output_quant_info.offset : b_const_int;
+ max_activation = act_info.activation() == ActivationLayerInfo::ActivationFunction::RELU ? 255 : a_const_int;
+ }
+ else
+ {
+ _fuse_activation = false;
+ }
}
// Set the GEMMLowp output stage info
@@ -323,7 +340,7 @@
// In case of NHWC, we need to run GEMM3D (gemm_3d_depth != 0) in order to avoid reshaping the output matrix
const unsigned int gemm_3d_depth = (data_layout == DataLayout::NHWC) ? conv_h : 0;
- configure_mm(gemm_input_to_use, &_weights_reshaped, biases, gemm_output_to_use, gemmlowp_output_stage, gemm_3d_depth);
+ configure_mm(gemm_input_to_use, &_weights_reshaped, biases_to_use, gemm_output_to_use, gemmlowp_output_stage, gemm_3d_depth, act_info);
if(!_skip_im2col)
{
@@ -345,7 +362,7 @@
ARM_COMPUTE_ERROR_ON_MSG((output->info()->dimension(idx_width) != conv_w) || (output->info()->dimension(idx_height) != conv_h),
"Output shape does not match the expected one");
- if(_is_activationlayer_enabled)
+ if(!_fuse_activation)
{
_activationlayer_function.configure(output, nullptr, act_info);
}
@@ -382,12 +399,10 @@
const ITensorInfo *gemm_output_to_use = output;
const ITensorInfo *weights_to_use = weights;
- const bool is_quantized = is_data_type_quantized_asymmetric(data_type);
- const bool append_bias = (biases != nullptr) && (!is_quantized);
- const bool skip_im2col = (data_layout == DataLayout::NHWC && kernel_width == 1 && kernel_height == 1 && conv_info.stride().first == 1 && conv_info.stride().second == 1);
- const bool skip_col2im = data_layout == DataLayout::NHWC;
- bool is_activationlayer_enabled = act_info.enabled();
- const bool run_addition = (skip_im2col) && (append_bias);
+ const bool is_quantized = is_data_type_quantized_asymmetric(data_type);
+ const bool skip_im2col = (data_layout == DataLayout::NHWC && kernel_width == 1 && kernel_height == 1 && conv_info.stride().first == 1 && conv_info.stride().second == 1);
+ const bool skip_col2im = data_layout == DataLayout::NHWC;
+ bool fuse_activation = true;
const UniformQuantizationInfo iq_info = input->quantization_info().uniform();
const UniformQuantizationInfo wq_info = weights->quantization_info().uniform();
@@ -429,10 +444,26 @@
unsigned int mat_weights_cols = weights->dimension(idx_kernels) / num_groups;
- // Output tensor auto inizialitation if not yet initialized
- ARM_COMPUTE_RETURN_ON_ERROR(CLConvolutionLayerReshapeWeights::validate(weights, is_quantized ? nullptr : biases, nullptr, num_groups));
- weights_reshaped_info = TensorInfo(compute_weights_reshaped_shape(*weights, (append_bias && !skip_im2col), num_groups), 1, data_type);
- weights_to_use = &weights_reshaped_info;
+ const ITensorInfo *biases_to_use = biases;
+ bool append_bias = false;
+
+ if(num_groups != 1 && biases != nullptr)
+ {
+ // num_groups != 1 can only be for NCHW
+ // Since it is missing an utility function to reshape the biases, we append the biases into the weights tensor
+ biases_to_use = nullptr;
+ append_bias = true;
+
+ ARM_COMPUTE_RETURN_ON_ERROR(CLConvolutionLayerReshapeWeights::validate(weights, biases, nullptr, num_groups));
+ weights_reshaped_info = TensorInfo(compute_weights_reshaped_shape(*weights, true, num_groups), 1, data_type);
+ }
+ else
+ {
+ ARM_COMPUTE_RETURN_ON_ERROR(CLConvolutionLayerReshapeWeights::validate(weights, nullptr, nullptr, num_groups));
+ weights_reshaped_info = TensorInfo(compute_weights_reshaped_shape(*weights, false, num_groups), 1, data_type);
+ }
+
+ weights_to_use = &weights_reshaped_info;
if(!skip_im2col)
{
@@ -446,11 +477,6 @@
ARM_COMPUTE_RETURN_ON_ERROR(CLIm2ColKernel::validate(input, &im2col_reshaped_info, kernel_dims, conv_info, append_bias, dilation, num_groups));
gemm_input_to_use = &im2col_reshaped_info;
}
- else if(run_addition)
- {
- // Validate add bias kernel
- ARM_COMPUTE_RETURN_ON_ERROR(CLSaturatedArithmeticOperationKernel::validate(ArithmeticOperation::ADD, output, biases, output, ConvertPolicy::SATURATE));
- }
// Create GEMM output tensor
if(!skip_col2im)
@@ -490,16 +516,20 @@
ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU
};
- if(is_activationlayer_enabled && supported_acts.count(act_info.activation()) != 0)
+ if(act_info.enabled())
{
- const int a_const_int = quantize_qasymm8(act_info.a(), output_quant_info);
- const int b_const_int = quantize_qasymm8(act_info.b(), output_quant_info);
+ if(supported_acts.count(act_info.activation()) != 0)
+ {
+ const int a_const_int = quantize_qasymm8(act_info.a(), output_quant_info);
+ const int b_const_int = quantize_qasymm8(act_info.b(), output_quant_info);
- min_activation = act_info.activation() != ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU ? output_quant_info.offset : b_const_int;
- max_activation = act_info.activation() == ActivationLayerInfo::ActivationFunction::RELU ? 255 : a_const_int;
-
- // If the activation layer is RELU, BOUNDED_RELU or LU_BOUNDED_RELU, we can use the GEMMLowp output stage to perform this operation
- is_activationlayer_enabled = false;
+ min_activation = act_info.activation() != ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU ? output_quant_info.offset : b_const_int;
+ max_activation = act_info.activation() == ActivationLayerInfo::ActivationFunction::RELU ? 255 : a_const_int;
+ }
+ else
+ {
+ fuse_activation = false;
+ }
}
// Set the GEMMLowp output stage info
@@ -513,7 +543,7 @@
// In case of NHWC, we need to run GEMM3D (gemm_3d_depth != 0) in order to avoid reshaping the output matrix
const unsigned int gemm_3d_depth = (data_layout == DataLayout::NHWC) ? conv_h : 0;
- ARM_COMPUTE_RETURN_ON_ERROR(validate_mm(gemm_input_to_use, weights_to_use, biases, gemm_output_to_use, gemmlowp_output_stage, gemm_3d_depth, skip_im2col, run_addition));
+ ARM_COMPUTE_RETURN_ON_ERROR(validate_mm(gemm_input_to_use, weights_to_use, biases_to_use, gemm_output_to_use, gemmlowp_output_stage, gemm_3d_depth, skip_im2col, act_info));
// Validate Col2Im
if(!skip_col2im)
@@ -522,7 +552,7 @@
}
//Validate Activation Layer
- if(is_activationlayer_enabled)
+ if(!fuse_activation)
{
ARM_COMPUTE_RETURN_ON_ERROR(CLActivationLayer::validate(output, nullptr, act_info));
}
@@ -554,19 +584,14 @@
_mm_gemm.run();
}
- if(_run_addition)
- {
- CLScheduler::get().enqueue(_add_bias_kernel);
- }
-
// Reshape output matrix
if(!_skip_col2im)
{
CLScheduler::get().enqueue(_col2im_kernel, false);
}
- //Run Activation Layer if enabled
- if(_is_activationlayer_enabled)
+ //Run Activation Layer if we cannot fuse in GEMM
+ if(!_fuse_activation)
{
_activationlayer_function.run();
}