COMPMID-519: Add support for Lower and Upper Bounded RELU for CL/NEON
Change-Id: I7b16216ac59c899a33942bf17757b54535256d7a
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/86172
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>
diff --git a/src/core/CL/cl_kernels/activation_layer.cl b/src/core/CL/cl_kernels/activation_layer.cl
index 119879a..4424a66 100644
--- a/src/core/CL/cl_kernels/activation_layer.cl
+++ b/src/core/CL/cl_kernels/activation_layer.cl
@@ -76,6 +76,11 @@
{
return min((TYPE)A_VAL, max(0, x));
}
+// Lower Upper Bounded RELU Activation
+inline TYPE lu_brelu_op(TYPE x)
+{
+ return min(max(x, (TYPE)B_VAL), (TYPE)A_VAL);
+}
// Leaky RELU Activation
inline TYPE lrelu_op(TYPE x)
{
diff --git a/src/core/NEON/kernels/NEActivationLayerKernel.cpp b/src/core/NEON/kernels/NEActivationLayerKernel.cpp
index 3195411..4ff26c0 100644
--- a/src/core/NEON/kernels/NEActivationLayerKernel.cpp
+++ b/src/core/NEON/kernels/NEActivationLayerKernel.cpp
@@ -73,6 +73,7 @@
{ ActivationFunction::LOGISTIC, &NEActivationLayerKernel::activation<ActivationFunction::LOGISTIC, float> },
{ ActivationFunction::RELU, &NEActivationLayerKernel::activation<ActivationFunction::RELU, float> },
{ ActivationFunction::BOUNDED_RELU, &NEActivationLayerKernel::activation<ActivationFunction::BOUNDED_RELU, float> },
+ { ActivationFunction::LU_BOUNDED_RELU, &NEActivationLayerKernel::activation<ActivationFunction::LU_BOUNDED_RELU, float> },
{ ActivationFunction::LEAKY_RELU, &NEActivationLayerKernel::activation<ActivationFunction::LEAKY_RELU, float> },
{ ActivationFunction::SOFT_RELU, &NEActivationLayerKernel::activation<ActivationFunction::SOFT_RELU, float> },
{ ActivationFunction::SQRT, &NEActivationLayerKernel::activation<ActivationFunction::SQRT, float> },
@@ -89,6 +90,7 @@
{ ActivationFunction::LOGISTIC, &NEActivationLayerKernel::activation<ActivationFunction::LOGISTIC, float16_t> },
{ ActivationFunction::RELU, &NEActivationLayerKernel::activation<ActivationFunction::RELU, float16_t> },
{ ActivationFunction::BOUNDED_RELU, &NEActivationLayerKernel::activation<ActivationFunction::BOUNDED_RELU, float16_t> },
+ { ActivationFunction::LU_BOUNDED_RELU, &NEActivationLayerKernel::activation<ActivationFunction::LU_BOUNDED_RELU, float16_t> },
{ ActivationFunction::SOFT_RELU, &NEActivationLayerKernel::activation<ActivationFunction::SOFT_RELU, float16_t> },
{ ActivationFunction::SQRT, &NEActivationLayerKernel::activation<ActivationFunction::SQRT, float16_t> },
{ ActivationFunction::SQUARE, &NEActivationLayerKernel::activation<ActivationFunction::SQUARE, float16_t> },
@@ -104,6 +106,7 @@
{ ActivationFunction::LOGISTIC, &NEActivationLayerKernel::activation<ActivationFunction::LOGISTIC, qint8_t> },
{ ActivationFunction::RELU, &NEActivationLayerKernel::activation<ActivationFunction::RELU, qint8_t> },
{ ActivationFunction::BOUNDED_RELU, &NEActivationLayerKernel::activation<ActivationFunction::BOUNDED_RELU, qint8_t> },
+ { ActivationFunction::LU_BOUNDED_RELU, &NEActivationLayerKernel::activation<ActivationFunction::LU_BOUNDED_RELU, qint8_t> },
{ ActivationFunction::LEAKY_RELU, &NEActivationLayerKernel::activation<ActivationFunction::LEAKY_RELU, qint8_t> },
{ ActivationFunction::SOFT_RELU, &NEActivationLayerKernel::activation<ActivationFunction::SOFT_RELU, qint8_t> },
{ ActivationFunction::SQRT, &NEActivationLayerKernel::activation<ActivationFunction::SQRT, qint8_t> },
@@ -118,6 +121,7 @@
{ ActivationFunction::LOGISTIC, &NEActivationLayerKernel::activation<ActivationFunction::LOGISTIC, qint16_t> },
{ ActivationFunction::RELU, &NEActivationLayerKernel::activation<ActivationFunction::RELU, qint16_t> },
{ ActivationFunction::BOUNDED_RELU, &NEActivationLayerKernel::activation<ActivationFunction::BOUNDED_RELU, qint16_t> },
+ { ActivationFunction::LU_BOUNDED_RELU, &NEActivationLayerKernel::activation<ActivationFunction::LU_BOUNDED_RELU, qint16_t> },
{ ActivationFunction::LEAKY_RELU, &NEActivationLayerKernel::activation<ActivationFunction::LEAKY_RELU, qint16_t> },
{ ActivationFunction::SOFT_RELU, &NEActivationLayerKernel::activation<ActivationFunction::SOFT_RELU, qint16_t> },
{ ActivationFunction::SQRT, &NEActivationLayerKernel::activation<ActivationFunction::SQRT, qint16_t> },
@@ -211,6 +215,15 @@
}
};
break;
+ case ActivationFunction::LU_BOUNDED_RELU:
+ tmp =
+ {
+ {
+ vminq_f16(a, vmaxq_f16(b, in.val[0])),
+ vminq_f16(a, vmaxq_f16(b, in.val[1]))
+ }
+ };
+ break;
case ActivationFunction::LINEAR:
tmp =
{
@@ -370,6 +383,17 @@
}
};
break;
+ case ActivationFunction::LU_BOUNDED_RELU:
+ tmp =
+ {
+ {
+ vminq_f32(a, vmaxq_f32(b, in.val[0])),
+ vminq_f32(a, vmaxq_f32(b, in.val[1])),
+ vminq_f32(a, vmaxq_f32(b, in.val[2])),
+ vminq_f32(a, vmaxq_f32(b, in.val[3])),
+ }
+ };
+ break;
case ActivationFunction::LEAKY_RELU:
tmp =
{
@@ -471,6 +495,9 @@
case ActivationFunction::BOUNDED_RELU:
tmp = vminq_qs8(a, vmaxq_qs8(CONST_0, in));
break;
+ case ActivationFunction::LU_BOUNDED_RELU:
+ tmp = vminq_qs8(a, vmaxq_qs8(b, in));
+ break;
case ActivationFunction::LEAKY_RELU:
tmp = vbslq_s8(vcgtq_s8(in, CONST_0), in, vmulq_qs8(a, in, fixed_point_position));
break;
@@ -562,6 +589,15 @@
}
};
break;
+ case ActivationFunction::LU_BOUNDED_RELU:
+ tmp =
+ {
+ {
+ vminq_qs16(a, vmaxq_qs16(b, in.val[0])),
+ vminq_qs16(a, vmaxq_qs16(b, in.val[1])),
+ }
+ };
+ break;
case ActivationFunction::LEAKY_RELU:
tmp =
{
diff --git a/src/core/Utils.cpp b/src/core/Utils.cpp
index cf8e194..66d3fe8 100644
--- a/src/core/Utils.cpp
+++ b/src/core/Utils.cpp
@@ -156,6 +156,7 @@
{ ActivationLayerInfo::ActivationFunction::LOGISTIC, "LOGISTIC" },
{ ActivationLayerInfo::ActivationFunction::RELU, "RELU" },
{ ActivationLayerInfo::ActivationFunction::BOUNDED_RELU, "BRELU" },
+ { ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, "LU_BRELU" },
{ ActivationLayerInfo::ActivationFunction::LEAKY_RELU, "LRELU" },
{ ActivationLayerInfo::ActivationFunction::SOFT_RELU, "SRELU" },
{ ActivationLayerInfo::ActivationFunction::SQRT, "SQRT" },