commit 146138378c1587b7297d245b7177641315f6180b
author Georgios Pinitas <georgios.pinitas@arm.com> Fri Mar 01 19:07:11 2019 +0000
committer Georgios Pinitas <georgios.pinitas@arm.com> Tue Mar 19 11:53:45 2019 +0000
tree 4689218b48a0884418099f373015429f2845ceaf
parent 3dd5b6884a65c06bcb9d15589ee2dc2978e3b336

COMPMID-1995: Update RSH GEMM assembly kernels.

-Updates u8/s8 hybrid dot product kernels to work for any N and any K >=16.
-Adds hybrid FP32 kernels with generic and A55 variants.
-Adds SVE native kernels for fp16/u8/s8.

Change-Id: Ifc0eaba9e3c8ea5bb19d334e870e1b39e4e7e728
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Reviewed-on: https://review.mlplatform.org/c/863
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Michele Di Giorgio <michele.digiorgio@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Michalis Spyrou <michalis.spyrou@arm.com>

src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp

diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
index b561659..0927123 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
@@ -28,49 +28,76 @@
 #include "arm_gemm.hpp"
 
 #include "gemm_common.hpp"
+#include "gemm_hybrid.hpp"
 #include "gemm_implementation.hpp"
 #include "gemm_interleaved.hpp"
+#include "gemm_native.hpp"
 
+#include "kernels/a32_sgemm_8x6.hpp"
 #include "kernels/a64_hgemm_24x8.hpp"
 #include "kernels/a64_sgemm_12x8.hpp"
-#include "kernels/a32_sgemm_8x6.hpp"
+#include "kernels/sve_hybrid_fp16_mla_4VLx4.hpp"
 #include "kernels/sve_interleaved_fp16_mla_3VLx8.hpp"
+#include "kernels/sve_native_fp16_mla_4VLx4.hpp"
 
 namespace arm_gemm {
 
 static const GemmImplementation<__fp16, __fp16> gemm_fp16_methods[] = {
 #if defined(__ARM_FEATURE_SVE)
 {
+    GemmMethod::GEMM_HYBRID,
+    "hybrid_fp16_mla_4VLx4",
+    [](const GemmArgs<__fp16> &args) { return (args._Ksize >= 8) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; },
+    [](const GemmArgs<__fp16> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+    [](const GemmArgs<__fp16> &args) { return new GemmHybrid<hybrid_fp16_mla_4VLx4, __fp16, __fp16>(args); }
+},
+{
+    GemmMethod::GEMM_NATIVE,
+    "native_fp16_mla_4VLx4",
+    [](const GemmArgs<__fp16> &args) { return (args._Ksize >= 8 && args._alpha==1.0f && !args._trA && !args._trB); },
+    [](const GemmArgs<__fp16> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+    [](const GemmArgs<__fp16> &args) { return new GemmNative<native_fp16_mla_4VLx4, __fp16, __fp16>(args); }
+},
+{
     GemmMethod::GEMM_INTERLEAVED,
     "interleaved_fp16_mla_3VLx8",
     [](const GemmArgs<__fp16> &args) { return (args._Ksize > 4); },
-    [](const GemmArgs<__fp16> &args) { return true; },
+    nullptr,
     [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<interleaved_fp16_mla_3VLx8, __fp16, __fp16>(args); }
 },
 #endif
+
 #if defined(__aarch64__) && (defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) || defined(FP16_KERNELS))
 {
     GemmMethod::GEMM_INTERLEAVED,
     "hgemm_24x8",
-    [](const GemmArgs<__fp16> &args) {
 #ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-        return args._ci->has_fp16();
+    [](const GemmArgs<__fp16> &args) { return args._ci->has_fp16(); },
 #else
-        return true;
+    nullptr,
 #endif
-    },
-    [](const GemmArgs<__fp16> &args) { return true; },
+    nullptr,
     [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<hgemm_24x8, __fp16, __fp16>(args); }
 },
 #endif
-#if defined(__arm__)
+#ifdef __aarch64__
+{
+    GemmMethod::GEMM_INTERLEAVED,
+    "sgemm_12x8",
+    nullptr,
+    nullptr,
+    [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<sgemm_12x8, __fp16, __fp16>(args); }
+},
+#elif defined(__arm__)
 {
     GemmMethod::GEMM_INTERLEAVED,
     "sgemm_8x6",
-    [](const GemmArgs<__fp16> &args) { return true; },
-    [](const GemmArgs<__fp16> &args) { return true; },
+    nullptr,
+    nullptr,
     [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<sgemm_8x6, __fp16, __fp16>(args); }
 },
+#else // not AArch64 or AArch32
+# error Unknown Architecture
 #endif
 {
     GemmMethod::DEFAULT,
@@ -90,8 +117,8 @@
 template UniqueGemmCommon<__fp16, __fp16> gemm<__fp16, __fp16>(const GemmArgs<__fp16> &args);
 template KernelDescription get_gemm_method<__fp16, __fp16>(const GemmArgs<__fp16> &args);
 template bool method_is_compatible<__fp16, __fp16>(GemmMethod method, const GemmArgs<__fp16> &args);
-template std::vector<std::string> get_compatible_kernels<__fp16, __fp16> (const GemmArgs<__fp16> &args);
+template std::vector<KernelDescription> get_compatible_kernels<__fp16, __fp16> (const GemmArgs<__fp16> &args);
 
 } // namespace arm_gemm
 
-#endif // __ARM_FP16_ARGS
\ No newline at end of file
+#endif // __ARM_FP16_ARGS