IVGCVSW-4439: Adding Elu support to Activation
* Added CpuRef implementation
* Added Unit Tests
* Added Quantizer Test
* Enabled Tests for Neon and CL backends on fp32 only
* Added to Serializer
Signed-off-by: David Monahan <david.monahan@arm.com>
Change-Id: Ic23e1797dbc9352b40678c389d7fe2b836b582ea
diff --git a/include/armnn/Types.hpp b/include/armnn/Types.hpp
index af331a2..b00a8bf 100644
--- a/include/armnn/Types.hpp
+++ b/include/armnn/Types.hpp
@@ -62,7 +62,8 @@
LeakyReLu = 6,
Abs = 7,
Sqrt = 8,
- Square = 9
+ Square = 9,
+ Elu = 10
};
enum class ArgMinMaxFunction
diff --git a/include/armnn/TypesUtils.hpp b/include/armnn/TypesUtils.hpp
index bf54c15..eca8a5e 100644
--- a/include/armnn/TypesUtils.hpp
+++ b/include/armnn/TypesUtils.hpp
@@ -38,6 +38,7 @@
case ActivationFunction::Abs: return "Abs";
case ActivationFunction::Sqrt: return "Sqrt";
case ActivationFunction::Square: return "Square";
+ case ActivationFunction::Elu: return "Elu";
default: return "Unknown";
}
}
diff --git a/src/armnn/test/QuantizerTest.cpp b/src/armnn/test/QuantizerTest.cpp
index faadf24..6d5d212 100644
--- a/src/armnn/test/QuantizerTest.cpp
+++ b/src/armnn/test/QuantizerTest.cpp
@@ -707,6 +707,62 @@
VisitLayersTopologically(quantizedNetworkQSymmS16.get(), validatorQSymmS16);
}
+
+BOOST_AUTO_TEST_CASE(QuantizeELuActivation)
+{
+ class TestEluActivationQuantization : public TestQuantization
+ {
+ public:
+ TestEluActivationQuantization(const TensorShape& inputShape, const TensorShape& outputShape)
+ : TestQuantization(inputShape, outputShape) {}
+
+ TestEluActivationQuantization(const QuantizerOptions& options,
+ const TensorShape& inputShape,
+ const TensorShape& outputShape)
+ : TestQuantization(options, inputShape, outputShape) {}
+
+ void VisitActivationLayer(const IConnectableLayer* layer,
+ const ActivationDescriptor& descriptor,
+ const char* name = nullptr) override
+ {
+ boost::ignore_unused(descriptor, name);
+ TensorInfo info = layer->GetOutputSlot(0).GetTensorInfo();
+
+ // Based off default static range [-15.0f, 15.0f]
+ TestQuantizationParams(
+ info, {30.0f / g_AsymmU8QuantizationBase, 128},
+ {30.0f / g_AsymmS8QuantizationBase, 0},
+ {15.0f / g_SymmS8QuantizationBase, 0},
+ {15.0f / g_SymmS16QuantizationBase, 0});
+ }
+ };
+
+ ActivationDescriptor descriptor;
+ descriptor.m_Function = ActivationFunction::Elu;
+
+ const TensorShape shape{1U};
+ INetworkPtr network = CreateNetworkWithActivationLayer(descriptor, shape);
+
+ INetworkPtr quantizedNetworkQAsymmU8 = INetworkQuantizer::Create(network.get())->ExportNetwork();
+ TestEluActivationQuantization validatorQAsymmU8(shape, shape);
+ VisitLayersTopologically(quantizedNetworkQAsymmU8.get(), validatorQAsymmU8);
+
+ const QuantizerOptions qAsymmS8Options(DataType::QAsymmS8);
+ INetworkPtr quantizedNetworkQAsymmS8 = INetworkQuantizer::Create(network.get(), qAsymmS8Options)->ExportNetwork();
+ TestEluActivationQuantization validatorQAsymmS8(qAsymmS8Options, shape, shape);
+ VisitLayersTopologically(quantizedNetworkQAsymmS8.get(), validatorQAsymmS8);
+
+ const QuantizerOptions qSymmS8Options(DataType::QSymmS8);
+ INetworkPtr quantizedNetworkQSymmS8 = INetworkQuantizer::Create(network.get(), qSymmS8Options)->ExportNetwork();
+ TestEluActivationQuantization validatorQSymmS8(qSymmS8Options, shape, shape);
+ VisitLayersTopologically(quantizedNetworkQSymmS8.get(), validatorQSymmS8);
+
+ const QuantizerOptions qSymmS16options(DataType::QSymmS16);
+ INetworkPtr quantizedNetworkQSymmS16 = INetworkQuantizer::Create(network.get(), qSymmS16options)->ExportNetwork();
+ TestEluActivationQuantization validatorQSymmS16(qSymmS16options, shape, shape);
+ VisitLayersTopologically(quantizedNetworkQSymmS16.get(), validatorQSymmS16);
+}
+
BOOST_AUTO_TEST_CASE(QuantizeBatchNorm)
{
class TestBatchNormalizationQuantization : public TestQuantization
diff --git a/src/armnnDeserializer/Deserializer.cpp b/src/armnnDeserializer/Deserializer.cpp
index 3bbd71a..61a38f9 100644
--- a/src/armnnDeserializer/Deserializer.cpp
+++ b/src/armnnDeserializer/Deserializer.cpp
@@ -421,6 +421,8 @@
return armnn::ActivationFunction::Sqrt;
case armnnSerializer::ActivationFunction_Square:
return armnn::ActivationFunction::Square;
+ case armnnSerializer::ActivationFunction_Elu:
+ return armnn::ActivationFunction::Elu;
default:
return armnn::ActivationFunction::Sigmoid;
}
diff --git a/src/armnnSerializer/ArmnnSchema.fbs b/src/armnnSerializer/ArmnnSchema.fbs
index be6616d..0697517 100644
--- a/src/armnnSerializer/ArmnnSchema.fbs
+++ b/src/armnnSerializer/ArmnnSchema.fbs
@@ -19,7 +19,8 @@
LeakyReLu = 6,
Abs = 7,
Sqrt = 8,
- Square = 9
+ Square = 9,
+ Elu = 10
}
enum ArgMinMaxFunction : byte {
diff --git a/src/armnnSerializer/Serializer.cpp b/src/armnnSerializer/Serializer.cpp
index b43f26c..3c01842 100644
--- a/src/armnnSerializer/Serializer.cpp
+++ b/src/armnnSerializer/Serializer.cpp
@@ -46,6 +46,8 @@
return serializer::ActivationFunction::ActivationFunction_Sqrt;
case armnn::ActivationFunction::Square:
return serializer::ActivationFunction::ActivationFunction_Square;
+ case armnn::ActivationFunction::Elu:
+ return serializer::ActivationFunction::ActivationFunction_Elu;
default:
return serializer::ActivationFunction::ActivationFunction_Sigmoid;
}
diff --git a/src/backends/aclCommon/ArmComputeUtils.hpp b/src/backends/aclCommon/ArmComputeUtils.hpp
index 4d69090..01a5445 100644
--- a/src/backends/aclCommon/ArmComputeUtils.hpp
+++ b/src/backends/aclCommon/ArmComputeUtils.hpp
@@ -65,6 +65,7 @@
case ActivationFunction::Sqrt: return AclActivationFunction::SQRT;
case ActivationFunction::Square: return AclActivationFunction::SQUARE;
case ActivationFunction::TanH: return AclActivationFunction::TANH;
+ case ActivationFunction::Elu: return AclActivationFunction::ELU;
default: throw InvalidArgumentException("Unsupported activation function");
}
}
diff --git a/src/backends/backendsCommon/test/layerTests/ActivationTestImpl.cpp b/src/backends/backendsCommon/test/layerTests/ActivationTestImpl.cpp
index 3f5e6c3..1b6e782 100644
--- a/src/backends/backendsCommon/test/layerTests/ActivationTestImpl.cpp
+++ b/src/backends/backendsCommon/test/layerTests/ActivationTestImpl.cpp
@@ -1016,6 +1016,64 @@
}
+template<armnn::DataType ArmnnType, typename T = armnn::ResolveType<ArmnnType>>
+LayerTestResult<T, 4> EluTestCommon(
+ armnn::IWorkloadFactory& workloadFactory,
+ const armnn::IBackendInternal::IMemoryManagerSharedPtr& memoryManager,
+ float qScale,
+ int32_t qOffset)
+{
+ std::vector<float> inputData = {
+ -0.1f, -0.2f, -0.3f, -0.4f,
+ 0.1f, 0.2f, 0.3f, 0.4f,
+ -1.0f, -2.0f, -3.0f, -4.0f,
+ 1.0f, 2.0f, 3.0f, 4.0f
+ };
+
+
+ const float a = 0.01f;
+ // Calculate output values for input.
+ auto f = [a](float value)
+ {
+ return (value >= 0) ? value : a * (expf(value) - 1);
+ };
+ std::vector<float> outputExpectedData(inputData.size());
+ std::transform(inputData.begin(), inputData.end(), outputExpectedData.begin(), f);
+
+ return SimpleActivationTest<ArmnnType>(workloadFactory,
+ memoryManager,
+ armnn::ActivationFunction::Elu,
+ a,
+ 0.0f,
+ qScale,
+ qOffset,
+ inputData,
+ qScale,
+ qOffset,
+ outputExpectedData);
+}
+
+LayerTestResult<float, 4> EluTest(
+ armnn::IWorkloadFactory& workloadFactory,
+ const armnn::IBackendInternal::IMemoryManagerSharedPtr& memoryManager)
+{
+ return EluTestCommon<armnn::DataType::Float32>(workloadFactory, memoryManager, 0.1f, 0);
+}
+
+LayerTestResult<uint8_t, 4> EluUint8Test(
+ armnn::IWorkloadFactory& workloadFactory,
+ const armnn::IBackendInternal::IMemoryManagerSharedPtr& memoryManager)
+{
+ return EluTestCommon<armnn::DataType::QAsymmU8>(workloadFactory, memoryManager, 0.1f, 64);
+}
+
+LayerTestResult<int16_t, 4> EluInt16Test(
+ armnn::IWorkloadFactory& workloadFactory,
+ const armnn::IBackendInternal::IMemoryManagerSharedPtr& memoryManager)
+{
+ return EluTestCommon<armnn::DataType::QSymmS16>(workloadFactory, memoryManager, 0.1f, 0);
+}
+
template<armnn::DataType ArmnnType, typename T = armnn::ResolveType<ArmnnType>>
LayerTestResult<T,4> CompareActivationTestImpl(
diff --git a/src/backends/backendsCommon/test/layerTests/ActivationTestImpl.hpp b/src/backends/backendsCommon/test/layerTests/ActivationTestImpl.hpp
index 2251ceb..2830118 100644
--- a/src/backends/backendsCommon/test/layerTests/ActivationTestImpl.hpp
+++ b/src/backends/backendsCommon/test/layerTests/ActivationTestImpl.hpp
@@ -149,6 +149,22 @@
const armnn::IBackendInternal::IMemoryManagerSharedPtr& memoryManager);
//
+// Elu
+//
+
+LayerTestResult<float, 4> EluTest(
+ armnn::IWorkloadFactory& workloadFactory,
+ const armnn::IBackendInternal::IMemoryManagerSharedPtr& memoryManager);
+
+LayerTestResult<uint8_t, 4> EluUint8Test(
+ armnn::IWorkloadFactory& workloadFactory,
+ const armnn::IBackendInternal::IMemoryManagerSharedPtr& memoryManager);
+
+LayerTestResult<int16_t, 4> EluInt16Test(
+ armnn::IWorkloadFactory& workloadFactory,
+ const armnn::IBackendInternal::IMemoryManagerSharedPtr& memoryManager);
+
+//
// Abs
//
diff --git a/src/backends/cl/test/ClLayerTests.cpp b/src/backends/cl/test/ClLayerTests.cpp
index 294c77b..cfec81a 100644
--- a/src/backends/cl/test/ClLayerTests.cpp
+++ b/src/backends/cl/test/ClLayerTests.cpp
@@ -39,6 +39,9 @@
ARMNN_AUTO_TEST_CASE(ReLu1Uint8, BoundedReLuUint8UpperAndLowerBoundTest)
ARMNN_AUTO_TEST_CASE(ReLu6Uint8, BoundedReLuUint8UpperBoundOnlyTest)
+// Elu Activation
+ARMNN_AUTO_TEST_CASE(Elu, EluTest)
+
ARMNN_AUTO_TEST_CASE(Logistic, SimpleSigmoidTest)
ARMNN_AUTO_TEST_CASE(LogisticUint8, SimpleSigmoidUint8Test)
diff --git a/src/backends/neon/test/NeonLayerTests.cpp b/src/backends/neon/test/NeonLayerTests.cpp
index 1a9d7a9..18658a3 100644
--- a/src/backends/neon/test/NeonLayerTests.cpp
+++ b/src/backends/neon/test/NeonLayerTests.cpp
@@ -418,6 +418,9 @@
ARMNN_AUTO_TEST_CASE(ReLu1Uint8, BoundedReLuUint8UpperAndLowerBoundTest)
ARMNN_AUTO_TEST_CASE(ReLu6Uint8, BoundedReLuUint8UpperBoundOnlyTest)
+// Elu Activation
+ARMNN_AUTO_TEST_CASE(Elu, EluTest)
+
// Sigmoid
ARMNN_AUTO_TEST_CASE(SimpleSigmoid, SimpleSigmoidTest)
ARMNN_AUTO_TEST_CASE(SimpleSigmoidUint8, SimpleSigmoidUint8Test)
diff --git a/src/backends/reference/RefLayerSupport.cpp b/src/backends/reference/RefLayerSupport.cpp
index 5cb36c4..8f1f170 100644
--- a/src/backends/reference/RefLayerSupport.cpp
+++ b/src/backends/reference/RefLayerSupport.cpp
@@ -108,6 +108,7 @@
{
case ActivationFunction::Abs:
case ActivationFunction::BoundedReLu:
+ case ActivationFunction::Elu:
case ActivationFunction::LeakyReLu:
case ActivationFunction::Linear:
case ActivationFunction::ReLu:
diff --git a/src/backends/reference/test/RefLayerTests.cpp b/src/backends/reference/test/RefLayerTests.cpp
index 99468e0..d5c67ef 100644
--- a/src/backends/reference/test/RefLayerTests.cpp
+++ b/src/backends/reference/test/RefLayerTests.cpp
@@ -462,6 +462,10 @@
ARMNN_AUTO_TEST_CASE(TanhUint8, TanhUint8Test)
ARMNN_AUTO_TEST_CASE(TanhInt16, TanhInt16Test)
+// Elu Activation
+ARMNN_AUTO_TEST_CASE(Elu, EluTest)
+ARMNN_AUTO_TEST_CASE(EluUint8, EluUint8Test)
+ARMNN_AUTO_TEST_CASE(EluInt16, EluInt16Test)
// Fully Connected
ARMNN_AUTO_TEST_CASE(SimpleFullyConnected, FullyConnectedFloat32Test, false, false)
diff --git a/src/backends/reference/workloads/Activation.cpp b/src/backends/reference/workloads/Activation.cpp
index 814a0dd..82dd919 100644
--- a/src/backends/reference/workloads/Activation.cpp
+++ b/src/backends/reference/workloads/Activation.cpp
@@ -69,6 +69,11 @@
output = a * tanhf(b * in);
break;
}
+ case ActivationFunction::Elu:
+ {
+ output = (in >= 0) ? in : a * (expf(in) - 1);
+ break;
+ }
default:
{
throw InvalidArgumentException("Unsupported activation function");