COMPMID-3776: Indirect GEMM
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Change-Id: I51a1b0f098bc3a8c408c50c92221e4df3061e12c
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/4343
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Sang-Hoon Park <sang-hoon.park@arm.com>
Reviewed-by: Michele Di Giorgio <michele.digiorgio@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
index ddb438f..e9e335f 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
@@ -24,6 +24,7 @@
#include "arm_gemm.hpp"
#include "gemm_common.hpp"
#include "gemm_hybrid.hpp"
+#include "gemm_hybrid_indirect.hpp"
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
#include "gemm_interleaved_pretransposed_2d.hpp"
@@ -31,127 +32,130 @@
#include "gemv_pretransposed.hpp"
#include "kernels/a32_sgemm_8x6.hpp"
-#include "kernels/a64_hybrid_fp32_mla_16x4.hpp"
-#include "kernels/a64_hybrid_fp32_mla_4x8.hpp"
-#include "kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp"
-#include "kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp"
-#include "kernels/a64_sgemm_12x8.hpp"
-#include "kernels/a64_sgemv_pretransposed.hpp"
+#include "kernels/a64_gemv_fp32_mla_32.hpp"
+#include "kernels/a64_hybrid_fp32_mla_6x16.hpp"
+#include "kernels/a64_hybrid_fp32_mla_8x4.hpp"
+#include "kernels/a64_sgemm_8x12.hpp"
+#include "kernels/a64_smallK_hybrid_fp32_mla_6x4.hpp"
+#include "kernels/a64_smallK_hybrid_fp32_mla_8x4.hpp"
-#include "kernels/sve_hybrid_fp32_mla_4VLx4.hpp"
-#include "kernels/sve_hybrid_fp32_mmla_4VLx4.hpp"
-#include "kernels/sve_interleaved_fp32_mla_3VLx8.hpp"
-#include "kernels/sve_interleaved_fp32_mmla_3VLx8.hpp"
-#include "kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp"
+#include "kernels/sve_gemv_fp32_mla_8VL.hpp"
+#include "kernels/sve_hybrid_fp32_mla_6x4VL.hpp"
+#include "kernels/sve_hybrid_fp32_mla_8x1VL.hpp"
+#include "kernels/sve_interleaved_fp32_mla_8x3VL.hpp"
+#include "kernels/sve_interleaved_fp32_mmla_8x3VL.hpp"
+#include "kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp"
namespace arm_gemm {
static const GemmImplementation<float, float> gemm_fp32_methods[] =
{
+// GEMV cases - starting with 'gemv_batched' wrapper to turn batched GEMV into GEMM.
{
GemmMethod::GEMV_BATCHED,
"gemv_batched",
- [](const GemmArgs &args) { return (args._Msize==1) && (args._nbatches>1); },
+ [](const GemmArgs &args) { return args._Msize==1 && args._nbatches>1 && !args._indirect_input; },
nullptr,
[](const GemmArgs &args) { return new GemvBatched<float, float>(args); }
},
#ifdef __aarch64__
-{
- GemmMethod::GEMV_PRETRANSPOSED,
- "sgemv_pretransposed",
- [](const GemmArgs &args) { return (args._Msize==1 && args._nbatches==1); },
- nullptr,
- [](const GemmArgs &args) { return new GemvPretransposed<sgemv_pretransposed, float, float>(args); }
-},
-#if defined(__ARM_FEATURE_SVE) && defined(MMLA_FP32)
+#ifdef __ARM_FEATURE_SVE
{
GemmMethod::GEMM_HYBRID,
- "hybrid_fp32_mmla_4VLx4",
- [](const GemmArgs &args) { return (args._Ksize >= 4); },
- [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mmla_4VLx4, float, float>(args); }
+ "sve_gemv_fp32_mla_8VL",
+ [](const GemmArgs &args) { return args._Msize==1 && args._nbatches==1 && !args._indirect_input; },
+ nullptr,
+ [](const GemmArgs &args) { return new GemvPretransposed<cls_sve_gemv_fp32_mla_8VL, float, float>(args); }
},
+#endif
+{
+ GemmMethod::GEMM_HYBRID,
+ "a64_gemv_fp32_mla_32",
+ [](const GemmArgs &args) { return args._Msize==1 && args._nbatches==1 && !args._indirect_input; },
+ nullptr,
+ [](const GemmArgs &args) { return new GemvPretransposed<cls_a64_gemv_fp32_mla_32, float, float>(args); }
+},
+
+// MMLA next due to higher throughput (SVE only)
+#if defined(__ARM_FEATURE_SVE) && defined(MMLA_FP32)
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_fp32_mmla_3VLx8",
+ "sve_interleaved_fp32_mmla_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp32_mmla_3VLx8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_fp32_mmla_8x3VL, float, float>(args); }
},
#endif // __ARM_FEATURE_SVE && MMLA_FP32
#ifdef __ARM_FEATURE_SVE
-// SVE smallk / hybrid methods
+// SVE smallk / hybrid methods
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_fp32_mla_1VLx8",
- [](const GemmArgs &args) { return (args._Ksize <= 24); },
+ "sve_smallK_hybrid_fp32_mla_8x1VL",
+ [](const GemmArgs &args) { return args._Ksize <= 24 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_1VLx8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_sve_smallK_hybrid_fp32_mla_8x1VL, float, float>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "hybrid_fp32_mla_4VLx4",
- [](const GemmArgs &args) { return (args._Ksize >= 4); },
+ "sve_hybrid_fp32_mla_8x1VL",
+ nullptr,
+ [](const GemmArgs &args) { return (args._Nsize < 12); },
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_sve_hybrid_fp32_mla_8x1VL, float, float>(args); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "sve_hybrid_fp32_mla_6x4VL",
+ nullptr,
[](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_4VLx4, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_sve_hybrid_fp32_mla_6x4VL, float, float>(args); }
},
#endif // __ARM_FEATURE_SVE
// NEON hybrid methods
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_fp32_mla_4x8",
- [](const GemmArgs &args) { return (args._Ksize <= 8) && (args._Nsize % 4)==0; },
+ "a64_smallK_hybrid_fp32_mla_8x4",
+ [](const GemmArgs &args) { return args._Ksize <= 8 && (args._Nsize % 4)==0 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_fp32_mla_8x4, float, float>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_fp32_mla_4x6",
- [](const GemmArgs &args) { return (args._Ksize > 8) && (args._Ksize <= 16) && (args._Nsize % 4)==0; },
+ "a64_smallK_hybrid_fp32_mla_6x4",
+ [](const GemmArgs &args) { return (args._Ksize > 8 && args._Ksize <= 16) && (args._Nsize % 4)==0 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x6, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_fp32_mla_6x4, float, float>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "hybrid_fp32_mla_4x8_normal",
- [](const GemmArgs &args) { return (args._Ksize >= 4); },
+ "a64_hybrid_fp32_mla_8x4",
+ nullptr,
[](const GemmArgs &args) { return (args._Nsize < 12); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_4x8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_fp32_mla_8x4, float, float>(args); }
},
GemmImplementation<float, float>::with_estimate(
GemmMethod::GEMM_HYBRID,
- "hybrid_fp32_mla_16x4",
- [](const GemmArgs &args) { return (args._Ksize >= 4); },
- [](const GemmArgs &args) { return GemmHybrid<hybrid_fp32_mla_16x4, float, float>::estimate_cycles(args, hybrid_fp32_mla_16x4::get_performance_parameters(args._ci)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_16x4, float, float>(args); }
+ "a64_hybrid_fp32_mla_6x16",
+ nullptr,
+ [](const GemmArgs &args) { return GemmHybridIndirect<cls_a64_hybrid_fp32_mla_6x16, float, float>::estimate_cycles(args, cls_a64_hybrid_fp32_mla_6x16::get_performance_parameters(args._ci)); },
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_fp32_mla_6x16, float, float>(args); }
),
-
#ifdef __ARM_FEATURE_SVE
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_fp32_mla_3VLx8",
+ "sve_interleaved_fp32_mla_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp32_mla_3VLx8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_fp32_mla_8x3VL, float, float>(args); }
},
#endif // __ARM_FEATURE_SVE
-// Pretranposed, 2D split
-GemmImplementation<float, float>::with_estimate(
- GemmMethod::GEMM_INTERLEAVED_2D,
- "sgemm_12x8_2d",
- nullptr,
- [](const GemmArgs &args) { return GemmInterleavedPretransposed2d<sgemm_12x8, float, float>::estimate_cycles(args, sgemm_12x8::get_performance_parameters(args._ci)); },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<sgemm_12x8, float, float>(args); }
-),
-// 1D split (with pretransposed or not)
GemmImplementation<float, float>::with_estimate(
GemmMethod::GEMM_INTERLEAVED,
- "sgemm_12x8_1d",
+ "a64_sgemm_8x12",
nullptr,
- [](const GemmArgs &args) { return GemmInterleaved<sgemm_12x8, float, float>::estimate_cycles(args, sgemm_12x8::get_performance_parameters(args._ci)); },
- [](const GemmArgs &args) { return new GemmInterleaved<sgemm_12x8, float, float>(args); }
+ [](const GemmArgs &args) { return GemmInterleaved<cls_a64_sgemm_8x12, float, float>::estimate_cycles(args, cls_a64_sgemm_8x12::get_performance_parameters(args._ci)); },
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_sgemm_8x12, float, float>(args); }
),
#endif // __aarch64__