diff --git a/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp b/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp
index da7e771..fe54590 100644
--- a/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp
+++ b/src/runtime/NEON/functions/NEDirectConvolutionLayer.cpp
@@ -28,14 +28,11 @@
 #include "arm_compute/core/Validate.h"
 #include "arm_compute/runtime/NEON/NEScheduler.h"
 
-#include <cmath>
-#include <tuple>
-
 namespace arm_compute
 {
 NEDirectConvolutionLayer::NEDirectConvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager)
     : _memory_group(std::move(memory_manager)), _output_stage_kernel(), _conv_kernel(), _input_border_handler(), _activationlayer_function(), _accumulator(), _has_bias(false),
-      _is_activationlayer_enabled(false), _dim_split(Window::DimZ)
+      _is_activationlayer_enabled(false), _dim_split(Window::DimZ), _is_padding_required()
 {
 }
 
@@ -59,9 +56,13 @@
     {
         _output_stage_kernel.configure(output, bias);
     }
+    _is_padding_required = !_conv_kernel.border_size().empty();
 
-    // Add zero padding XY
-    _input_border_handler.configure(input, _conv_kernel.border_size(), BorderMode::CONSTANT, PixelValue(static_cast<float>(0.f)));
+    if(_is_padding_required)
+    {
+        // Add zero padding XY
+        _input_border_handler.configure(input, _conv_kernel.border_size(), BorderMode::CONSTANT, PixelValue(static_cast<float>(0.f)));
+    }
 
     //Configure Activation Layer
     _is_activationlayer_enabled = act_info.enabled();
@@ -104,10 +105,12 @@
 
 void NEDirectConvolutionLayer::run()
 {
-    NEScheduler::get().schedule(&_input_border_handler, Window::DimZ);
-
     MemoryGroupResourceScope scope_mg(_memory_group);
 
+    if(_is_padding_required)
+    {
+        NEScheduler::get().schedule(&_input_border_handler, Window::DimZ);
+    }
     NEScheduler::get().schedule(&_conv_kernel, _dim_split);
     if(_has_bias)
     {