COMPMID-3144: Remove padding from NEDirectConvolutionLayerKernel

Change-Id: I22b907eebfbe037e6e1c7bf604172f4709a9cbed
Signed-off-by: Manuel Bottini <manuel.bottini@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/4082
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Michele Di Giorgio <michele.digiorgio@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>
diff --git a/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp b/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp
index da7e771..fe54590 100644
--- a/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp
+++ b/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp
@@ -28,14 +28,11 @@
 #include "arm_compute/core/Validate.h"
 #include "arm_compute/runtime/NEON/NEScheduler.h"
 
-#include <cmath>
-#include <tuple>
-
 namespace arm_compute
 {
 NEDirectConvolutionLayer::NEDirectConvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager)
     : _memory_group(std::move(memory_manager)), _output_stage_kernel(), _conv_kernel(), _input_border_handler(), _activationlayer_function(), _accumulator(), _has_bias(false),
-      _is_activationlayer_enabled(false), _dim_split(Window::DimZ)
+      _is_activationlayer_enabled(false), _dim_split(Window::DimZ), _is_padding_required()
 {
 }
 
@@ -59,9 +56,13 @@
     {
         _output_stage_kernel.configure(output, bias);
     }
+    _is_padding_required = !_conv_kernel.border_size().empty();
 
-    // Add zero padding XY
-    _input_border_handler.configure(input, _conv_kernel.border_size(), BorderMode::CONSTANT, PixelValue(static_cast<float>(0.f)));
+    if(_is_padding_required)
+    {
+        // Add zero padding XY
+        _input_border_handler.configure(input, _conv_kernel.border_size(), BorderMode::CONSTANT, PixelValue(static_cast<float>(0.f)));
+    }
 
     //Configure Activation Layer
     _is_activationlayer_enabled = act_info.enabled();
@@ -104,10 +105,12 @@
 
 void NEDirectConvolutionLayer::run()
 {
-    NEScheduler::get().schedule(&_input_border_handler, Window::DimZ);
-
     MemoryGroupResourceScope scope_mg(_memory_group);
 
+    if(_is_padding_required)
+    {
+        NEScheduler::get().schedule(&_input_border_handler, Window::DimZ);
+    }
     NEScheduler::get().schedule(&_conv_kernel, _dim_split);
     if(_has_bias)
     {