COMPMID-1867: Add NEON/SVE GEMM Hybrid kernels.
Change-Id: Ib40a9921e7f9a6a8be6c38872d6b3a0f24ed0cd3
Reviewed-on: https://review.mlplatform.org/515
Reviewed-by: Anthony Barbier <Anthony.barbier@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
index 9194bdd..1a90e96 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -38,75 +38,51 @@
namespace arm_gemm {
-#ifdef __ARM_FEATURE_SVE
-class GemmImpl_gemm_fp16_interleaved_fp16 : public GemmImplementation<__fp16, __fp16> {
-public:
-
- UniqueGemmCommon<__fp16, __fp16> instantiate(const GemmArgs<__fp16> &args) override {
- return UniqueGemmCommon<__fp16, __fp16>(new GemmInterleaved<interleaved_fp16_mla_3VLx8, __fp16, __fp16>(args));
- }
-
- GemmImpl_gemm_fp16_interleaved_fp16() : GemmImplementation<__fp16, __fp16>(GemmMethod::GEMM_INTERLEAVED_FP16) { }
-};
-
-#elif defined(__aarch64__)
-
-#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) || defined(FP16_KERNELS)
-class GemmImpl_gemm_fp16_interleaved_fp16 : public GemmImplementation<__fp16, __fp16> {
-public:
+static const GemmImplementation<__fp16, __fp16> gemm_fp16_methods[] = {
+#if defined(__ARM_FEATURE_SVE)
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "interleaved_fp16_mla_3VLx8",
+ [](const GemmArgs<__fp16> &args) { return (args._Ksize > 4); },
+ [](const GemmArgs<__fp16> &args) { return true; },
+ [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<interleaved_fp16_mla_3VLx8, __fp16, __fp16>(args); }
+},
+#endif
+#if defined(__aarch64__) && (defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) || defined(FP16_KERNELS))
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "hgemm_24x8",
+ [](const GemmArgs<__fp16> &args) {
#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
- bool is_supported(const GemmArgs<__fp16> &args) override {
return args._ci->has_fp16();
- }
-#endif
-
- UniqueGemmCommon<__fp16, __fp16> instantiate(const GemmArgs<__fp16> &args) override {
- return UniqueGemmCommon<__fp16, __fp16>(new GemmInterleaved<hgemm_24x8, __fp16, __fp16>(args));
- }
-
- GemmImpl_gemm_fp16_interleaved_fp16() : GemmImplementation<__fp16, __fp16>(GemmMethod::GEMM_INTERLEAVED_FP16) { }
-};
-#endif
-
-#endif // __aarch64__
-
-class GemmImpl_gemm_fp16_interleaved : public GemmImplementation<__fp16, __fp16> {
-public:
- UniqueGemmCommon<__fp16, __fp16> instantiate(const GemmArgs<__fp16> &args) override {
-#ifdef __aarch64__
- return UniqueGemmCommon<__fp16, __fp16>(new GemmInterleaved<sgemm_12x8, __fp16, __fp16>(args));
-#elif defined(__arm__)
- return UniqueGemmCommon<__fp16, __fp16>(new GemmInterleaved<sgemm_8x6, __fp16, __fp16>(args));
#else
-# error Unknown Architecture
+ return true;
#endif
- }
-
- GemmImpl_gemm_fp16_interleaved() : GemmImplementation<__fp16, __fp16>(GemmMethod::GEMM_INTERLEAVED) { }
-};
-
-#if defined(__aarch64__) && (defined(__ARM_FEATURE_VECTOR_ARITHMETIC) || defined(FP16_KERNELS) || defined(__ARM_FEATURE_SVE))
-static GemmImpl_gemm_fp16_interleaved_fp16 gemm_fp16_interleaved_fp16_impl{};
+ },
+ [](const GemmArgs<__fp16> &args) { return true; },
+ [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<hgemm_24x8, __fp16, __fp16>(args); }
+},
#endif
-static GemmImpl_gemm_fp16_interleaved gemm_fp16_interleaved_impl{};
-
-static std::vector<GemmImplementation<__fp16, __fp16> *> gemm_fp16_methods = {
-#if defined(__aarch64__) && (defined(__ARM_FEATURE_VECTOR_ARITHMETIC) || defined(FP16_KERNELS) || defined(__ARM_FEATURE_SVE))
- &gemm_fp16_interleaved_fp16_impl,
-#endif
- &gemm_fp16_interleaved_impl
+{
+ GemmMethod::DEFAULT,
+ "",
+ nullptr,
+ nullptr,
+ nullptr,
+}
};
template<>
-std::vector<GemmImplementation<__fp16, __fp16> *> &gemm_implementation_list<__fp16, __fp16>() {
+const GemmImplementation<__fp16, __fp16> *gemm_implementation_list<__fp16, __fp16>() {
return gemm_fp16_methods;
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<__fp16, __fp16> gemm<__fp16, __fp16>(GemmArgs<__fp16> &args, GemmConfig *cfg);
-template GemmMethod get_gemm_method<__fp16, __fp16>(GemmArgs<__fp16> &args);
-template bool method_is_compatible<__fp16, __fp16>(GemmMethod method, GemmArgs<__fp16> &args);
+template UniqueGemmCommon<__fp16, __fp16> gemm<__fp16, __fp16>(const GemmArgs<__fp16> &args);
+template KernelDescription get_gemm_method<__fp16, __fp16>(const GemmArgs<__fp16> &args);
+template bool method_is_compatible<__fp16, __fp16>(GemmMethod method, const GemmArgs<__fp16> &args);
+template std::vector<std::string> get_compatible_kernels<__fp16, __fp16> (const GemmArgs<__fp16> &args);
} // namespace arm_gemm
-#endif // __ARM_FP16_ARGS
+#endif // __ARM_FP16_ARGS
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
index 7d14971..8bc33cc 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -23,6 +23,7 @@
*/
#include "arm_gemm.hpp"
#include "gemm_common.hpp"
+#include "gemm_hybrid.hpp"
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
#include "gemm_native.hpp"
@@ -30,112 +31,140 @@
#include "gemv_native_transposed.hpp"
#include "gemv_pretransposed.hpp"
-#include "kernels/a64_sgemm_12x8.hpp"
#include "kernels/a32_sgemm_8x6.hpp"
-#include "kernels/a64_sgemv_trans.hpp"
-#include "kernels/a64_sgemv_pretransposed.hpp"
+#include "kernels/a64_sgemm_12x8.hpp"
#include "kernels/a64_sgemm_native_16x4.hpp"
+#include "kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp"
+#include "kernels/a64_sgemv_pretransposed.hpp"
+#include "kernels/a64_sgemv_trans.hpp"
+#include "kernels/sve_hybrid_fp32_mla_4VLx4.hpp"
#include "kernels/sve_interleaved_fp32_mla_3VLx8.hpp"
+#include "kernels/sve_native_fp32_mla_4VLx4.hpp"
+#include "kernels/sve_smallK_fp32_mla_1VLx4.hpp"
+#include "kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp"
namespace arm_gemm {
-#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE)
-// SGEMM implementations for AArch64 without SVE
+static const GemmImplementation<float, float> gemm_fp32_methods[] =
+{
+{
+ GemmMethod::GEMV_BATCHED,
+ "gemv_batched",
+ [](const GemmArgs<float> &args) { return (args._Msize==1) && (args._nbatches>1); },
+ nullptr,
+ [](const GemmArgs<float> &args) { return new GemvBatched<float, float>(args); }
+},
+#ifdef __aarch64__
+{
+ GemmMethod::GEMV_PRETRANSPOSED,
+ "sgemv_pretransposed",
+ [](const GemmArgs<float> &args) { return (args._Msize==1 && args._alpha==1.0f && args._pretransposed_hint && args._nbatches==1); },
+ nullptr,
+ [](const GemmArgs<float> &args) { return new GemvPretransposed<sgemv_pretransposed, float, float>(args); }
+},
+{
+ GemmMethod::GEMV_NATIVE_TRANSPOSED,
+ "sgemv_trans",
+ [](const GemmArgs<float> &args) { return (args._Msize==1 && args._alpha==1.0f && !args._trA && !args._trB && args._nbatches==1); },
+ nullptr,
+ [](const GemmArgs<float> &args) { return new GemvNativeTransposed<sgemv_trans, float, float>(args); }
+},
-// Pretransposed GEMV
-class GemmImpl_sgemm_gemv_pretransposed : public GemmImplementation<float, float> {
-public:
- bool is_supported(const GemmArgs<float> &args) override {
- return (args._Msize==1 && args._alpha==1.0f && args._pretransposed_hint && args._nbatches==1);
- }
+#ifdef __ARM_FEATURE_SVE
+ // SVE smallk / native / hybrid methods
+{
+ GemmMethod::GEMM_HYBRID,
+ "smallK_hybrid_fp32_mla_1VLx4",
+ [](const GemmArgs<float> &args) { return (args._Ksize <= 24) && !args._trA && args._alpha==1.0f && args._pretransposed_hint; },
+ nullptr,
+ [](const GemmArgs<float> &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_1VLx4, float, float>(args); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "hybrid_fp32_mla_4VLx4",
+ [](const GemmArgs<float> &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs<float> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs<float> &args) { return new GemmHybrid<hybrid_fp32_mla_4VLx4, float, float>(args); }
+},
+{
+ GemmMethod::GEMM_NATIVE,
+ "smallK_fp32_mla_1VLx4",
+ [](const GemmArgs<float> &args) { return (args._Ksize <= 24) && !args._trA && !args._trB && args._alpha==1.0f; },
+ nullptr,
+ [](const GemmArgs<float> &args) { return new GemmNative<smallK_fp32_mla_1VLx4, float, float>(args); }
+},
+{
+ GemmMethod::GEMM_NATIVE,
+ "native_fp32_mla_4VLx4",
+ [](const GemmArgs<float> &args) { return (args._Ksize>4 && args._alpha==1.0f && !args._trA && !args._trB); },
+ [](const GemmArgs<float> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs<float> &args) { return new GemmNative<native_fp32_mla_4VLx4, float, float>(args); }
+},
+#endif // __ARM_FEATURE_SVE
- UniqueGemmCommon<float, float> instantiate(const GemmArgs<float> &args) override {
- return UniqueGemmCommon<float, float> (new GemvPretransposed<sgemv_pretransposed, float, float>(args._ci, args._Nsize, args._Ksize, args._nmulti, args._trB, args._beta));
- }
+// NEON native / hybrid methods
+{
+ GemmMethod::GEMM_HYBRID,
+ "sgemm_nativeA_pretransposeB_16x4",
+ [](const GemmArgs<float> &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; },
+ [](const GemmArgs<float> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs<float> &args) { return new GemmHybrid<sgemm_nativeA_pretransposeB_16x4, float, float>(args); }
+},
+{
+ GemmMethod::GEMM_NATIVE,
+ "sgemm_native_16x4",
+ [](const GemmArgs<float> &args) { return (args._Ksize>4 && (args._Nsize % 16)==0 && args._alpha==1.0f && !args._trA && !args._trB); },
+ [](const GemmArgs<float> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs<float> &args) { return new GemmNative<sgemm_native_16x4, float, float>(args); }
+},
- GemmImpl_sgemm_gemv_pretransposed() : GemmImplementation<float, float>(GemmMethod::GEMV_PRETRANSPOSED) { }
-};
-
-// Native GEMV
-class GemmImpl_sgemm_gemv_native_transposed : public GemmImplementation<float, float> {
-public:
- bool is_supported(const GemmArgs<float> &args) override {
- return (args._Msize==1 && args._alpha==1.0f && !args._trA && !args._trB && args._nbatches==1);
- }
-
- UniqueGemmCommon<float, float> instantiate(const GemmArgs<float> &args) override {
- return UniqueGemmCommon<float, float> (new GemvNativeTransposed<sgemv_trans, float, float>(args._ci, args._Nsize, args._Ksize, args._nmulti, args._beta));
- }
-
- GemmImpl_sgemm_gemv_native_transposed() : GemmImplementation<float, float>(GemmMethod::GEMV_NATIVE_TRANSPOSED) { }
-};
-
-// Native GEMM
-class GemmImpl_sgemm_gemm_native : public GemmImplementation<float, float> {
-public:
- bool is_supported(const GemmArgs<float> &args) override {
- return (args._Ksize>4 && (args._Nsize % 16)==0 && args._alpha==1.0f && !args._trA && !args._trB);
- }
-
- bool is_recommended(const GemmArgs<float> &args) override {
- return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8));
- }
-
- UniqueGemmCommon<float, float> instantiate(const GemmArgs<float> &args) override {
- return UniqueGemmCommon<float, float> (new GemmNative<sgemm_native_16x4, float, float>(args._ci, args._Msize, args._Nsize, args._Ksize, args._nbatches, args._nmulti, args._beta));
- }
-
- GemmImpl_sgemm_gemm_native() : GemmImplementation<float, float>(GemmMethod::GEMM_NATIVE) { }
-};
+#ifdef __ARM_FEATURE_SVE
+ {
+ GemmMethod::GEMM_INTERLEAVED,
+ "interleaved_fp32_mla_3VLx8",
+ [](const GemmArgs<float> &args) { return (args._Ksize>4); },
+ nullptr,
+ [](const GemmArgs<float> &args) { return new GemmInterleaved<interleaved_fp32_mla_3VLx8, float, float>(args); }
+},
+#endif // __ARM_FEATURE_SVE
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "sgemm_12x8",
+ nullptr,
+ nullptr,
+ [](const GemmArgs<float> &args) { return new GemmInterleaved<sgemm_12x8, float, float>(args); }
+},
#endif // __aarch64__
-// Interleaved GEMM
-class GemmImpl_sgemm_gemm_interleaved : public GemmImplementation<float, float> {
-public:
- UniqueGemmCommon<float, float> instantiate(const GemmArgs<float> &args) override {
-#ifdef __ARM_FEATURE_SVE
- return UniqueGemmCommon<float, float> (new GemmInterleaved<interleaved_fp32_mla_3VLx8, float, float>(args));
-#elif defined(__aarch64__)
- return UniqueGemmCommon<float, float> (new GemmInterleaved<sgemm_12x8, float, float>(args));
-#elif defined(__arm__)
- return UniqueGemmCommon<float, float> (new GemmInterleaved<sgemm_8x6, float, float>(args));
-#else
-# error Unknown Architecture.
-#endif
- }
-
- GemmImpl_sgemm_gemm_interleaved() : GemmImplementation<float, float>(GemmMethod::GEMM_INTERLEAVED) { }
-};
-
-static GemmImpl_gemv_batched<float, float> gemv_batched_impl{};
-#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE)
-static GemmImpl_sgemm_gemv_pretransposed sgemm_gemv_pretransposed_impl{};
-static GemmImpl_sgemm_gemv_native_transposed sgemm_gemv_native_transposed_impl{};
-static GemmImpl_sgemm_gemm_native sgemm_gemm_native_impl{};
-#endif
-static GemmImpl_sgemm_gemm_interleaved sgemm_gemm_interleaved_impl{};
-
-/* List of implementations (order matters) */
-static std::vector<GemmImplementation<float, float> *> SGemmMethods = {
- &gemv_batched_impl,
-#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE)
- &sgemm_gemv_pretransposed_impl,
- &sgemm_gemv_native_transposed_impl,
- &sgemm_gemm_native_impl,
-#endif
- &sgemm_gemm_interleaved_impl
+#ifdef __arm__
+ {
+ GemmMethod::GEMM_INTERLEAVED,
+ "sgemm_8x6",
+ nullptr,
+ nullptr,
+ [](const GemmArgs<float> &args) { return new GemmInterleaved<sgemm_8x6, float, float>(args); }
+},
+#endif // __arm__
+{
+ GemmMethod::DEFAULT,
+ "",
+ nullptr,
+ nullptr,
+ nullptr
+}
};
/* Templated function to return this list. */
template<>
-std::vector<GemmImplementation<float, float> *> &gemm_implementation_list<float, float>() {
- return SGemmMethods;
+const GemmImplementation<float, float> *gemm_implementation_list<float, float>() {
+ return gemm_fp32_methods;
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<float, float> gemm<float, float>(GemmArgs<float> &args, GemmConfig *cfg);
-template GemmMethod get_gemm_method<float, float>(GemmArgs<float> &args);
-template bool method_is_compatible<float, float>(GemmMethod method, GemmArgs<float> &args);
+template UniqueGemmCommon<float, float> gemm<float, float>(const GemmArgs<float> &args);
+template KernelDescription get_gemm_method<float, float>(const GemmArgs<float> &args);
+template bool method_is_compatible<float, float>(GemmMethod method, const GemmArgs<float> &args);
+template std::vector<std::string> get_compatible_kernels<float, float> (const GemmArgs<float> &args);
-} // namespace arm_gemm
+} // namespace arm_gemm
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp
new file mode 100644
index 0000000..09f03c6
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp
@@ -0,0 +1,303 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#include <assert.h>
+
+#include <algorithm>
+
+#include "arm_gemm.hpp"
+#include "utils.hpp"
+
+#include "mergeresults.hpp"
+#include "transform.hpp"
+
+#ifdef CYCLE_PROFILING
+#include "profiler.hpp"
+#endif
+
+namespace arm_gemm {
+
+// Implementation of the GemmCommon abstract class.
+template<typename strategy, typename To, typename Tr>
+class GemmHybrid : public GemmCommon<To, Tr> {
+ typedef typename strategy::operand_type Toi;
+ typedef typename strategy::result_type Tri;
+
+ /* const properties set by constructor */
+ const CPUInfo * const _ci;
+
+ const unsigned int _Msize;
+ const unsigned int _Nsize;
+ const unsigned int _Ksize;
+
+ const unsigned int _nbatches;
+ const unsigned int _nmulti;
+
+ const bool _trB;
+
+ const Tr _beta;
+
+ /* Blocking info */
+ unsigned int _k_block=0;
+ unsigned int _x_block=0;
+ unsigned int _Mround=0;
+
+ /* Pretransposed buffer. */
+ const Toi *_B_transposed=nullptr;
+
+ unsigned int _B_per_multi = 0;
+
+ /* We will need to walk through the blocks of B in a few contexts, so
+ * factor that out. */
+ class blockwalker {
+ private:
+ /* Size loops, etc. based on our parent's configuration */
+ const GemmHybrid<strategy, To, Tr> &_parent;
+
+ /* K, X and multi parameters for current iteration. */
+ unsigned int _k0=0, _x0=0;
+
+ unsigned int _index=0;
+ bool _done=false;
+ bool _newkblock=true;
+
+ public:
+ blockwalker(const GemmHybrid<strategy, To, Tr> &parent) : _parent(parent) { }
+
+ unsigned int xmax() {
+ return std::min(_x0 + _parent._x_block, _parent._Nsize);
+ }
+
+ unsigned int kmax() {
+ return std::min(_k0 + _parent._k_block, _parent._Ksize);
+ }
+
+ /* Advance to the next block, return false at the end. */
+ bool advance(void) {
+ if (_done) {
+ return false;
+ }
+
+ _newkblock=false;
+ _x0 += _parent._x_block;
+ if (_x0 >= _parent._Nsize) {
+ _x0=0;
+ _k0 += _parent._k_block;
+ if (_k0 >= _parent._Ksize) {
+ _done=true;
+ return false;
+ }
+ _newkblock=true;
+ }
+ _index++;
+
+ return true;
+ }
+
+ unsigned int k0(void) { return _k0; }
+ unsigned int x0(void) { return _x0; }
+ unsigned int index(void) { return _index; }
+ bool done(void) { return _done; }
+ bool newkblock(void) { return _newkblock; }
+ };
+
+
+public:
+ GemmHybrid(GemmHybrid &) = delete;
+ GemmHybrid & operator= (GemmHybrid &) = delete;
+
+ /* Constructor */
+ GemmHybrid(const GemmArgs<Tr> &args)
+ : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), _nbatches(args._nbatches),
+ _nmulti(args._nmulti), _trB(args._trB), _beta(args._beta) {
+ const unsigned int L1_size = _ci->get_L1_cache_size();
+ const unsigned int L2_size = _ci->get_L2_cache_size();
+
+ _B_per_multi = (iceildiv(_Nsize, strategy::out_width()) * strategy::out_width()) *
+ (iceildiv(_Ksize, strategy::k_unroll()) * strategy::k_unroll());
+
+ // Work out blocking parameters, or override from config.
+
+ if (args._cfg && args._cfg->inner_block_size) {
+ _k_block = args._cfg->inner_block_size;
+ } else {
+ // k_block: Find out how much of the larger array can be loaded into half the cache.
+ // This should account for associative caches.
+ _k_block = (L1_size / 2) / (sizeof(Toi) * (std::max(strategy::out_width(), strategy::out_height())));
+
+ // Needs to be (at least a single) multiple of the K unroll level.
+ _k_block /= strategy::k_unroll();
+ _k_block = std::max(_k_block, 1U) * strategy::k_unroll();
+
+ // Now tune to presented problem size; this is how many blocks we need.
+ int num_k_blocks = iceildiv(_Ksize, _k_block);
+
+ // So divide the space equally into that many blocks.
+ _k_block = iceildiv(_Ksize, num_k_blocks);
+
+ // And round UP to the K unroll level required.
+ _k_block = iceildiv(_k_block, strategy::k_unroll());
+ _k_block *= strategy::k_unroll();
+ }
+
+ if (args._cfg && args._cfg->outer_block_size) {
+ _x_block = args._cfg->outer_block_size;
+ } else {
+ // x_block: Work out how many rows (of length k_block) will fit in the L2
+ // Don't allocate more than 90% of the L2 to allow for overheads, and subtract off the L1 contents.
+ _x_block = (((L2_size * 9) / 10) - (_k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height()))) /
+ (sizeof(Toi) * _k_block);
+
+ // Needs to be (at least a single) multiple of the kernel output width.
+ _x_block /= strategy::out_width();
+ _x_block = std::max(_x_block, 1U) * strategy::out_width();
+
+ // And tune to the presented problem size.
+ int num_x_blocks = iceildiv(_Nsize, _x_block);
+ _x_block = iceildiv(_Nsize, num_x_blocks);
+
+ _x_block = iceildiv(_x_block, strategy::out_width());
+ _x_block *= strategy::out_width();
+ }
+
+ // Work out the rounded size of M - needed for some buffers.
+ _Mround = iceildiv(_Msize, strategy::out_height());
+ _Mround *= strategy::out_height();
+ }
+
+ // Interface implementation - Compulsory functions
+
+ // Window size: Only the last thread should do a ragged block, so dole
+ // out work in units of out_height. Factor batches and multi into the
+ // window too.
+ unsigned int get_window_size() const override {
+ // _Mround is a multiple of out_height by definition.
+ return (_Mround / strategy::out_height()) * _nbatches * _nmulti;
+ }
+
+ // Execute
+ void execute(unsigned int start, unsigned int end, int threadid) override {
+#ifdef CYCLE_PROFILING
+ profiler prof;
+#endif
+ strategy strat(_ci);
+
+ /* Make sure we've been set up correctly. */
+ assert(_B_transposed);
+
+ const unsigned int window_per_batch = iceildiv(_Msize, strategy::out_height());
+ const unsigned int window_per_multi = window_per_batch * _nbatches;
+
+ const unsigned int first_multi = start / window_per_multi;
+ const unsigned int last_multi = end / window_per_multi;
+
+ const unsigned int first_batch = (start - (first_multi * window_per_multi)) / window_per_batch;
+ const unsigned int last_batch = (end - (last_multi * window_per_multi)) / window_per_batch;
+
+ const unsigned int first_row = ((start - (first_multi * window_per_multi)) % window_per_batch) * strategy::out_height();
+ const unsigned int last_row = ((end - (last_multi * window_per_multi)) % window_per_batch) * strategy::out_height();
+
+ static_assert(std::is_same<To, Toi>::value, "gemm_native: Operand types must be the same.");
+ static_assert(std::is_same<Tr, Tri>::value, "gemm_native: Result types must be the same.");
+
+ for (unsigned int multi = first_multi; multi <= last_multi; multi++) {
+ const unsigned int batch_0 = (multi == first_multi) ? first_batch : 0;
+ const unsigned int batch_max = (multi == last_multi) ? last_batch : (_nbatches - 1);
+
+ const Toi *b_panel = _B_transposed + (multi * _B_per_multi);
+
+ for (blockwalker current(*this); !current.done(); current.advance()) {
+ int kern_k = iceildiv(current.kmax() - current.k0(), strategy::k_unroll());
+ kern_k *= strat.k_unroll();
+
+ int bblocks = iceildiv(current.xmax() - current.x0(), strategy::out_width());
+
+ for (unsigned int batch = batch_0; batch <= batch_max; batch++) {
+ const unsigned int m_start = ((multi == first_multi) && (batch == first_batch)) ? first_row : 0;
+ const unsigned int m_end = ((multi == last_multi) && (batch == last_batch) ) ? last_row : _Msize;
+#ifdef CYCLE_PROFILING
+ auto p = prof.ScopedProfiler(PROFILE_KERNEL, (m_end - m_start) * kern_k * bblocks * strategy::out_width());
+#endif
+
+ strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (m_start * this->_lda) + current.k0(), this->_lda,
+ b_panel,
+ this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (m_start * this->_ldc) + current.x0(), this->_ldc,
+ (current.k0() == 0) ? _beta : static_cast<Tr>(1),
+ (m_end - m_start), (current.xmax() - current.x0()), kern_k);
+ }
+
+ b_panel += (bblocks * strat.out_width() * kern_k);
+ }
+ }
+ }
+
+ // Interface implementation - pretransposed
+ bool B_is_pretransposed() const override {
+ return true;
+ }
+
+ bool B_pretranspose_required() const override {
+ return (_B_transposed==nullptr);
+ }
+
+ size_t get_B_pretransposed_array_size() const override {
+ return _B_per_multi * _nmulti * sizeof(Toi);
+ }
+
+ void pretranspose_B_array(void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override {
+ Toi *buffer = reinterpret_cast<Toi *>(in_buffer);
+ _B_transposed = buffer;
+ strategy strat(_ci);
+
+ for (unsigned int multi=0; multi < _nmulti; multi++) {
+ blockwalker current(*this);
+
+ do {
+ /* Figure out the size of each block. */
+ size_t x_size = (current.xmax() - current.x0());
+ size_t k_size = (current.kmax() - current.k0());
+
+ /* Round sizes up as needed. */
+ x_size = iceildiv(x_size, strategy::out_width());
+ x_size *= strategy::out_width();
+
+ k_size = iceildiv(k_size, strategy::k_unroll());
+ k_size *= strategy::k_unroll();
+
+ strat.transforms.PrepareB(
+ buffer, B + (multi * B_multi_stride), ldb,
+ current.x0(), current.xmax(), current.k0(), current.kmax(), _trB);
+
+ buffer += (x_size * k_size);
+ } while (current.advance());
+ }
+ }
+
+ void set_pretransposed_B_data(void *in_buffer) override {
+ _B_transposed = reinterpret_cast<Toi *>(in_buffer);
+ }
+};
+
+} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp
index 6734e3c..bf80784 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 ARM Limited.
+ * Copyright (c) 2018-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -22,56 +22,53 @@
* SOFTWARE.
*/
-#include "gemv_batched.hpp"
+#include <arm_gemm.hpp>
+
+#include <functional>
namespace arm_gemm {
template<typename Top, typename Tret>
-class GemmImplementation {
-public:
- /* Is this implementation compatible with the args as provided? */
- virtual bool is_supported(const GemmArgs<Tret> &args) { return true; }
- /* Is this implementation "recommended" for these args (heuristic)? */
- virtual bool is_recommended(const GemmArgs<Tret> &args) { return true; }
- /* Instantiate this method please. */
- virtual UniqueGemmCommon<Top, Tret> instantiate(const GemmArgs<Tret> &args) = 0;
-
- /* Indicate the "GemmMethod" for use as a selector */
- const GemmMethod method;
-
- virtual ~GemmImplementation() { }
-
- GemmImplementation(GemmMethod method) : method(method) { }
-};
-
-/* "gemv_batched" implementation is type-agnostic, so template it here. */
-template<typename Top, typename Tret>
-class GemmImpl_gemv_batched : public GemmImplementation<Top, Tret> {
-public:
- bool is_supported(const GemmArgs<Tret> &args) override {
- return (args._Msize==1 && args._nbatches > 1);
- }
-
- UniqueGemmCommon<Top, Tret> instantiate(const GemmArgs<Tret> &args) override {
- return UniqueGemmCommon<Top, Tret> (new GemvBatched<Top, Tret>(args));
- }
-
- GemmImpl_gemv_batched() : GemmImplementation<Top, Tret>(GemmMethod::GEMV_BATCHED) { }
+struct GemmImplementation {
+ const GemmMethod method;
+ const char * name;
+ std::function<bool(const GemmArgs<Tret> &)> is_supported;
+ std::function<bool(const GemmArgs<Tret> &)> is_recommended;
+ std::function<GemmCommon<Top, Tret> *(const GemmArgs<Tret> &)> instantiate;
};
/* "Master" function implemented for each valid combination of types.
* Returns a list of GEMM implementation descriptors for processing by the
- * other functions. */
+ * other functions, terminated by an implementation with
+ * method==GemmMethod::DEFAULT. */
template<typename Top, typename Tret>
-std::vector<GemmImplementation<Top, Tret> *> &gemm_implementation_list();
+const GemmImplementation<Top, Tret> *gemm_implementation_list();
+/*
+ * Select a GEMM implementation for the given arguments.
+ *
+ * The logic here returns the first method on the list which supports the
+ * requested problem parameters, matches the provided filters (method and/or
+ * name string match) and recommends itself.
+ *
+ * If there is no such method, it will return the first method which
+ * supports the requested parameters and passes the filters, regardless of
+ * recommendation.
+ *
+ * If no method supports the requested parameters and passes the filters,
+ * this function returns false and doesn't touch the provided pointer
+ * reference.
+ */
template<typename Top, typename Tret>
-GemmImplementation<Top, Tret> *find_implementation(GemmArgs<Tret> &args, GemmConfig *cfg) {
+bool find_implementation(const GemmArgs<Tret> &args, const GemmImplementation<Top, Tret> * &impl) {
auto gemms = gemm_implementation_list<Top, Tret>();
+ const GemmConfig *cfg = args._cfg;
- for(auto &&i : gemms) {
+ const GemmImplementation<Top, Tret> *saved_impl = nullptr;
+
+ for (auto i = gemms; i->method != GemmMethod::DEFAULT; i++) {
/* Skip if this implementation doesn't support these args. */
- if (!i->is_supported(args)) {
+ if (i->is_supported != nullptr && !i->is_supported(args)) {
continue;
}
@@ -80,52 +77,92 @@
continue;
}
- /* If no specific method is requested, check that this method recommends itself. */
- if ((!cfg || cfg->method == GemmMethod::DEFAULT) && !i->is_recommended(args)) {
+ /* Skip if a filter is to be applied and it doesn't match. */
+ if (cfg && cfg->filter != "" && !strstr(i->name, cfg->filter.c_str())) {
continue;
}
- return i;
+ /* At this point, if we don't have a saved implementation, save this
+ * one. This is so that we always return something if a filter
+ * matches, even if it doesn't recommend itself.
+ */
+ if (saved_impl == nullptr) {
+ saved_impl=i;
+ }
+
+ /* Check that this method recommends itself. */
+ if (i->is_recommended != nullptr && !i->is_recommended(args)) {
+ continue;
+ }
+
+ impl=i;
+
+ return true;
}
- return nullptr;
-}
-
-template<typename Top, typename Tret>
-UniqueGemmCommon<Top, Tret> gemm(GemmArgs<Tret> &args, GemmConfig *cfg) {
- auto impl = find_implementation<Top, Tret>(args, cfg);
-
- if (impl) {
- return impl->instantiate(args);
- }
-
- return UniqueGemmCommon<Top, Tret>(nullptr);
-}
-
-template<typename Top, typename Tret>
-GemmMethod get_gemm_method(GemmArgs<Tret> &args) {
- auto impl = find_implementation<Top, Tret>(args, nullptr);
-
- if (impl) {
- return impl->method;
- }
-
- /* This shouldn't happen - there should always be at least one valid implementation. */
- return GemmMethod::DEFAULT;
-}
-
-template<typename Top, typename Tret>
-bool method_is_compatible(GemmMethod method, GemmArgs<Tret> &args) {
- /* Determine if the method is valid by attempting to obtain an implementation specifying this method. */
- GemmConfig cfg(method);
-
- auto impl = find_implementation<Top, Tret>(args, &cfg);
-
- if (impl) {
+ /* We didn't find an option matching the filters that recommended
+ * itself. But if we found something earlier that matched the filters
+ * but wasn't recommended, return it here. */
+ if (saved_impl != nullptr) {
+ impl = saved_impl;
return true;
}
return false;
}
-} // namespace arm_gemm
+template<typename Top, typename Tret>
+std::vector<std::string> get_compatible_kernels(const GemmArgs<Tret> &args) {
+ std::vector<std::string> res;
+
+ auto gemms = gemm_implementation_list<Top, Tret>();
+
+ for (auto i = gemms; i->method != GemmMethod::DEFAULT; i++) {
+ /* Check that this implementation supports the presented problem. */
+ if (i->is_supported != nullptr && !i->is_supported(args)) {
+ continue;
+ }
+
+ res.push_back(i->name);
+ }
+
+ return res;
+}
+
+template<typename Top, typename Tret>
+UniqueGemmCommon<Top, Tret> gemm(const GemmArgs<Tret> &args) {
+ const GemmImplementation<Top, Tret> *impl;
+
+ if (find_implementation<Top, Tret>(args, impl)) {
+ return UniqueGemmCommon<Top, Tret>(impl->instantiate(args));
+ }
+
+ return UniqueGemmCommon<Top, Tret>(nullptr);
+}
+
+template<typename Top, typename Tret>
+KernelDescription get_gemm_method(const GemmArgs<Tret> &args) {
+ const GemmImplementation<Top, Tret> *impl;
+
+ if (find_implementation<Top, Tret>(args, impl)) {
+ return KernelDescription(impl->method, impl->name);
+ }
+
+ /* This shouldn't happen - there should always be at least one valid implementation. */
+ return KernelDescription();
+}
+
+template<typename Top, typename Tret>
+bool method_is_compatible(GemmMethod method, const GemmArgs<Tret> &args) {
+ /* Determine if the method is valid by attempting to obtain an implementation specifying this method. */
+ GemmConfig cfg(method);
+ GemmArgs<Tret> myargs = args;
+
+ myargs._cfg = &cfg;
+
+ const GemmImplementation<Top, Tret> *impl;
+
+ return find_implementation<Top, Tret>(myargs, impl);
+}
+
+} // namespace arm_gemm
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp
index ad171a7..b4503dd 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,30 +32,33 @@
namespace arm_gemm {
-class GemmImpl_gemm_s16_interleaved : public GemmImplementation<int16_t, int32_t> {
-public:
- UniqueGemmCommon<int16_t, int32_t> instantiate(const GemmArgs<int32_t> &args) override {
- return UniqueGemmCommon<int16_t, int32_t>(new GemmInterleaved<gemm_s16_12x8, int16_t, int32_t>(args));
- }
-
- GemmImpl_gemm_s16_interleaved() : GemmImplementation<int16_t, int32_t>(GemmMethod::GEMM_INTERLEAVED) { }
-};
-
-static GemmImpl_gemm_s16_interleaved gemm_s16_interleaved_impl{};
-
-static std::vector<GemmImplementation<int16_t, int32_t> *> gemm_s16_methods = {
- &gemm_s16_interleaved_impl
+static const GemmImplementation<int16_t, int32_t> gemm_s16_methods[] = {
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "gemm_s16_12x8",
+ nullptr,
+ nullptr,
+ [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<gemm_s16_12x8, int16_t, int32_t>(args); }
+},
+{
+ GemmMethod::DEFAULT,
+ "",
+ nullptr,
+ nullptr,
+ nullptr
+}
};
template<>
-std::vector<GemmImplementation<int16_t, int32_t> *> &gemm_implementation_list<int16_t, int32_t>() {
+const GemmImplementation<int16_t, int32_t> *gemm_implementation_list<int16_t, int32_t>() {
return gemm_s16_methods;
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<int16_t, int32_t> gemm<int16_t, int32_t>(GemmArgs<int32_t> &args, GemmConfig *cfg);
-template GemmMethod get_gemm_method<int16_t, int32_t>(GemmArgs<int32_t> &args);
-template bool method_is_compatible<int16_t, int32_t>(GemmMethod method, GemmArgs<int32_t> &args);
+template UniqueGemmCommon<int16_t, int32_t> gemm<int16_t, int32_t>(const GemmArgs<int32_t> &args);
+template KernelDescription get_gemm_method<int16_t, int32_t>(const GemmArgs<int32_t> &args);
+template bool method_is_compatible<int16_t, int32_t>(GemmMethod method, const GemmArgs<int32_t> &args);
+template std::vector<std::string> get_compatible_kernels<int16_t, int32_t> (const GemmArgs<int32_t> &args);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
index 627d8ab..34dc8bc 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -27,67 +27,67 @@
#include "gemm_common.hpp"
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
+#include "gemm_native.hpp"
#include "kernels/a64_gemm_s16_12x8.hpp"
#include "kernels/a64_gemm_s8_12x8.hpp"
#include "kernels/a64_gemm_s8_4x4.hpp"
#include "kernels/sve_interleaved_s8s32_dot_3VLx8.hpp"
+#include "kernels/sve_native_s8s32_dot_4VLx4.hpp"
namespace arm_gemm {
+static const GemmImplementation<int8_t, int32_t> gemm_s8_methods[] = {
#ifdef __ARM_FEATURE_SVE
-class GemmImpl_gemm_s8_interleaved_dot : public GemmImplementation<int8_t, int32_t> {
-public:
- UniqueGemmCommon<int8_t, int32_t> instantiate(const GemmArgs<int32_t> &args) override {
- return UniqueGemmCommon<int8_t, int32_t>(new GemmInterleaved<interleaved_s8s32_dot_3VLx8, int8_t, int32_t>(args));
- }
-
- GemmImpl_gemm_s8_interleaved_dot() : GemmImplementation<int8_t, int32_t>(GemmMethod::GEMM_INTERLEAVED_DOT) { }
-};
-#else
-
-class GemmImpl_gemm_s8_interleaved_dot : public GemmImplementation<int8_t, int32_t> {
-public:
- bool is_supported(const GemmArgs<int32_t> &args) override {
- return args._ci->has_dotprod();
- }
-
- UniqueGemmCommon<int8_t, int32_t> instantiate(const GemmArgs<int32_t> &args) override {
- return UniqueGemmCommon<int8_t, int32_t>(new GemmInterleaved<gemm_s8_12x8, int8_t, int32_t>(args));
- }
-
- GemmImpl_gemm_s8_interleaved_dot() : GemmImplementation<int8_t, int32_t>(GemmMethod::GEMM_INTERLEAVED_DOT) { }
-};
-
+{
+ GemmMethod::GEMM_NATIVE,
+ "native_s8s32_dot_4VLx4",
+ [](const GemmArgs<int32_t> &args) { return (args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB); },
+ [](const GemmArgs<int32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); },
+ [](const GemmArgs<int32_t> &args) { return new GemmNative<native_s8s32_dot_4VLx4, int8_t, int32_t>(args); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "interleaved_s8s32_dot_3VLx8",
+ [](const GemmArgs<int32_t> &args) { return (args._Ksize>4); },
+ nullptr,
+ [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<interleaved_s8s32_dot_3VLx8, int8_t, int32_t>(args); }
+},
#endif
-
-class GemmImpl_gemm_s8_interleaved : public GemmImplementation<int8_t, int32_t> {
-public:
- UniqueGemmCommon<int8_t, int32_t> instantiate(const GemmArgs<int32_t> &args) override {
- return UniqueGemmCommon<int8_t, int32_t>(new GemmInterleaved<gemm_s8_4x4, int8_t, int32_t>(args));
- }
-
- GemmImpl_gemm_s8_interleaved() : GemmImplementation<int8_t, int32_t>(GemmMethod::GEMM_INTERLEAVED) { }
-};
-
-static GemmImpl_gemm_s8_interleaved_dot gemm_s8_interleaved_dot_impl{};
-static GemmImpl_gemm_s8_interleaved gemm_s8_interleaved_impl{};
-
-static std::vector<GemmImplementation<int8_t, int32_t> *> gemm_s8_methods = {
- &gemm_s8_interleaved_dot_impl,
- &gemm_s8_interleaved_impl
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "gemm_s8_12x8",
+ [](const GemmArgs<int32_t> &args) { return args._ci->has_dotprod(); },
+ nullptr,
+ [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<gemm_s8_12x8, int8_t, int32_t>(args); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "gemm_s8_4x4",
+ nullptr,
+ nullptr,
+ [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<gemm_s8_4x4, int8_t, int32_t>(args); }
+},
+{
+ GemmMethod::DEFAULT,
+ "",
+ nullptr,
+ nullptr,
+ nullptr
+}
};
template<>
-std::vector<GemmImplementation<int8_t, int32_t> *> &gemm_implementation_list<int8_t, int32_t>() {
+const GemmImplementation<int8_t, int32_t> *gemm_implementation_list<int8_t, int32_t>() {
return gemm_s8_methods;
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<int8_t, int32_t> gemm<int8_t, int32_t>(GemmArgs<int32_t> &args, GemmConfig *cfg);
-template GemmMethod get_gemm_method<int8_t, int32_t>(GemmArgs<int32_t> &args);
-template bool method_is_compatible<int8_t, int32_t>(GemmMethod method, GemmArgs<int32_t> &args);
+template UniqueGemmCommon<int8_t, int32_t> gemm<int8_t, int32_t>(const GemmArgs<int32_t> &args);
+template KernelDescription get_gemm_method<int8_t, int32_t>(const GemmArgs<int32_t> &args);
+template bool method_is_compatible<int8_t, int32_t>(GemmMethod method, const GemmArgs<int32_t> &args);
+template std::vector<std::string> get_compatible_kernels<int8_t, int32_t> (const GemmArgs<int32_t> &args);
} // namespace arm_gemm
-#endif // __aarch64__
+#endif // __aarch64__
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp
index 0e58a4d..436438f 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -318,50 +318,57 @@
/* Constructor */
GemmInterleaved(const GemmArgs<Tr> &args)
- : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
- _nbatches(args._nbatches), _nmulti(args._nmulti), _trA(args._trA), _trB(args._trB),
- _alpha(args._alpha), _beta(args._beta), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
- _pretransposed(args._pretransposed_hint) {
+ : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
+ _nbatches(args._nbatches), _nmulti(args._nmulti), _trA(args._trA), _trB(args._trB),
+ _alpha(args._alpha), _beta(args._beta), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
+ _pretransposed(args._pretransposed_hint) {
const unsigned int L1_size = _ci->get_L1_cache_size();
const unsigned int L2_size = _ci->get_L2_cache_size();
assert(_maxthreads > 0);
- // Work out blocking parameters
+ // Work out blocking parameters, or override from provided GemmConfig
+ if (args._cfg && args._cfg->inner_block_size) {
+ _k_block = args._cfg->inner_block_size;
+ } else {
+ // k_block: Find out how much of the larger array can be loaded into half the cache.
+ // This should account for associative caches.
+ _k_block = (L1_size / 2) / (sizeof(Toi) * (std::max(strategy::out_width(), strategy::out_height())));
- // k_block: Find out how much of the larger array can be loaded into half the cache.
- // This should account for associative caches.
- _k_block = (L1_size / 2) / (sizeof(Toi) * (std::max(strategy::out_width(), strategy::out_height())));
+ // Needs to be (at least a single) multiple of the K unroll level.
+ _k_block /= strategy::k_unroll();
+ _k_block = std::max(_k_block, 1U) * strategy::k_unroll();
- // Needs to be (at least a single) multiple of the K unroll level.
- _k_block /= strategy::k_unroll();
- _k_block = std::max(_k_block, 1U) * strategy::k_unroll();
+ // Now tune to presented problem size; this is how many blocks we need.
+ int num_k_blocks = iceildiv(_Ksize, _k_block);
- // Now tune to presented problem size; this is how many blocks we need.
- int num_k_blocks = iceildiv(_Ksize, _k_block);
+ // So divide the space equally into that many blocks.
+ _k_block = iceildiv(_Ksize, num_k_blocks);
- // So divide the space equally into that many blocks.
- _k_block = iceildiv(_Ksize, num_k_blocks);
+ // And round UP to the K unroll level required.
+ _k_block = iceildiv(_k_block, strategy::k_unroll());
+ _k_block *= strategy::k_unroll();
+ }
- // And round UP to the K unroll level required.
- _k_block = iceildiv(_k_block, strategy::k_unroll());
- _k_block *= strategy::k_unroll();
+ if (args._cfg && args._cfg->outer_block_size) {
+ _x_block = args._cfg->outer_block_size;
+ } else {
+ // x_block: Work out how many rows (of length k_block) will fit in the L2
+ // Don't allocate more than 90% of the L2 to allow for overheads, and subtract off the L1 contents.
+ _x_block = (((L2_size * 9) / 10) - (_k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height()))) /
+ (sizeof(Toi) * _k_block);
- // x_block: Work out how many rows (of length k_block) will fit in the L2
- // Don't allocate more than 90% of the L2 to allow for overheads, and subtract off the L1 contents.
- _x_block = (((L2_size * 9) / 10) - (_k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height()))) /
- (sizeof(Toi) * _k_block);
+ // Needs to be (at least a single) multiple of the kernel output width.
+ _x_block /= strategy::out_width();
+ _x_block = std::max(_x_block, 1U) * strategy::out_width();
- // Needs to be (at least a single) multiple of the kernel output width.
- _x_block /= strategy::out_width();
- _x_block = std::max(_x_block, 1U) * strategy::out_width();
+ // And tune to the presented problem size.
+ int num_x_blocks = iceildiv(_Nsize, _x_block);
+ _x_block = iceildiv(_Nsize, num_x_blocks);
- // And tune to the presented problem size.
- int num_x_blocks = iceildiv(_Nsize, _x_block);
- _x_block = iceildiv(_Nsize, num_x_blocks);
-
- _x_block = iceildiv(_x_block, strategy::out_width());
- _x_block *= strategy::out_width();
+ _x_block = iceildiv(_x_block, strategy::out_width());
+ _x_block *= strategy::out_width();
+ }
// Work out the rounded size of M - needed for some buffers.
_Mround = iceildiv(_Msize, strategy::out_height());
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_native.hpp b/src/core/NEON/kernels/arm_gemm/gemm_native.hpp
index baa1316..5795334 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_native.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_native.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -74,11 +74,11 @@
GemmNative(GemmNative &) = delete;
GemmNative & operator= (GemmNative &) = delete;
- GemmNative(const CPUInfo *ci, const unsigned int M, const unsigned int N, const unsigned int K, const unsigned int nbatches, const unsigned int nmultis, const Tr beta) :
- _Msize(M), _Nsize(N), _Ksize(K), _nbatches(nbatches), _nmultis(nmultis), _beta(beta), _ci(ci) {
+ GemmNative(const GemmArgs<Tr> &args)
+ : _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), _nbatches(args._nbatches), _nmultis(args._nmulti), _beta(args._beta), _ci(args._ci) {
/* For now don't do any blocking. TODO: figure out if we should. */
- k_block = K;
- n_block = N;
+ k_block = _Ksize;
+ n_block = _Nsize;
}
// Window is amount per multi multiplied by total number of multis.
@@ -105,8 +105,13 @@
unsigned int y0 = batch_pos * strategy::out_height();
- for (unsigned int pos=start; pos<end; pos++) {
- const unsigned int ymax = std::min(y0 + strategy::out_height(), _Msize);
+ for (unsigned int l=end-start; l>0; ) {
+ // Do work from here to the end of the current batch/multi
+ const unsigned int ymax = std::min(y0 + (l * strategy::out_height()), _Msize);
+
+ // Work out how many units this is and subtract from loop counter.
+ l -= ((ymax - y0) + (strategy::out_height() - 1)) / strategy::out_height();
+
#ifdef CYCLE_PROFILING
auto p = prof.ScopedProfiler(PROFILE_KERNEL, (ymax-y0) * _Nsize * _Ksize);
#endif
@@ -117,7 +122,7 @@
_beta, (ymax-y0), _Nsize, _Ksize);
/* Advance to next item */
- y0 += strategy::out_height();
+ y0 = ymax;
/* Check for batch/multi overflow */
if (y0 >= _Msize) {
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp
index feea482..6bcbca9 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,31 +32,34 @@
namespace arm_gemm {
-class GemmImpl_gemm_u16_interleaved : public GemmImplementation<uint16_t, uint32_t> {
-public:
- UniqueGemmCommon<uint16_t, uint32_t> instantiate(const GemmArgs<uint32_t> &args) override {
- return UniqueGemmCommon<uint16_t, uint32_t>(new GemmInterleaved<gemm_u16_12x8, uint16_t, uint32_t>(args));
- }
-
- GemmImpl_gemm_u16_interleaved() : GemmImplementation<uint16_t, uint32_t>(GemmMethod::GEMM_INTERLEAVED) { }
-};
-
-static GemmImpl_gemm_u16_interleaved gemm_u16_interleaved_impl{};
-
-static std::vector<GemmImplementation<uint16_t, uint32_t> *> gemm_u16_methods = {
- &gemm_u16_interleaved_impl
+static const GemmImplementation<uint16_t, uint32_t> gemm_u16_methods[] = {
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "gemm_u16_12x8",
+ nullptr,
+ nullptr,
+ [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<gemm_u16_12x8, uint16_t, uint32_t>(args); }
+},
+{
+ GemmMethod::DEFAULT,
+ "",
+ nullptr,
+ nullptr,
+ nullptr
+}
};
template<>
-std::vector<GemmImplementation<uint16_t, uint32_t> *> &gemm_implementation_list<uint16_t, uint32_t>() {
+const GemmImplementation<uint16_t, uint32_t> *gemm_implementation_list<uint16_t, uint32_t>() {
return gemm_u16_methods;
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<uint16_t, uint32_t> gemm<uint16_t, uint32_t>(GemmArgs<uint32_t> &args, GemmConfig *cfg);
-template GemmMethod get_gemm_method<uint16_t, uint32_t>(GemmArgs<uint32_t> &args);
-template bool method_is_compatible<uint16_t, uint32_t>(GemmMethod method, GemmArgs<uint32_t> &args);
+template UniqueGemmCommon<uint16_t, uint32_t> gemm<uint16_t, uint32_t>(const GemmArgs<uint32_t> &args);
+template KernelDescription get_gemm_method<uint16_t, uint32_t>(const GemmArgs<uint32_t> &args);
+template bool method_is_compatible<uint16_t, uint32_t>(GemmMethod method, const GemmArgs<uint32_t> &args);
+template std::vector<std::string> get_compatible_kernels<uint16_t, uint32_t> (const GemmArgs<uint32_t> &args);
} // namespace arm_gemm
-#endif // __aarch64__
+#endif // __aarch64__
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
index b7c1bab..3c8df3f 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -27,64 +27,66 @@
#include "gemm_common.hpp"
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
+#include "gemm_native.hpp"
#include "kernels/a64_gemm_u16_12x8.hpp"
#include "kernels/a64_gemm_u8_12x8.hpp"
#include "kernels/a64_gemm_u8_4x4.hpp"
#include "kernels/sve_interleaved_u8u32_dot_3VLx8.hpp"
+#include "kernels/sve_native_u8u32_dot_4VLx4.hpp"
namespace arm_gemm {
+static const GemmImplementation<uint8_t, uint32_t> gemm_u8_methods[] = {
#ifdef __ARM_FEATURE_SVE
-class GemmImpl_gemm_u8_interleaved_dot : public GemmImplementation<uint8_t, uint32_t> {
-public:
- UniqueGemmCommon<uint8_t, uint32_t> instantiate(const GemmArgs<uint32_t> &args) override {
- return UniqueGemmCommon<uint8_t, uint32_t>(new GemmInterleaved<interleaved_u8u32_dot_3VLx8, uint8_t, uint32_t>(args));
- }
-
- GemmImpl_gemm_u8_interleaved_dot() : GemmImplementation<uint8_t, uint32_t>(GemmMethod::GEMM_INTERLEAVED_DOT) { }
-};
-#else
-class GemmImpl_gemm_u8_interleaved_dot : public GemmImplementation<uint8_t, uint32_t> {
-public:
- bool is_supported(const GemmArgs<uint32_t> &args) override {
- return args._ci->has_dotprod();
- }
-
- UniqueGemmCommon<uint8_t, uint32_t> instantiate(const GemmArgs<uint32_t> &args) override {
- return UniqueGemmCommon<uint8_t, uint32_t>(new GemmInterleaved<gemm_u8_12x8, uint8_t, uint32_t>(args));
- }
-
- GemmImpl_gemm_u8_interleaved_dot() : GemmImplementation<uint8_t, uint32_t>(GemmMethod::GEMM_INTERLEAVED_DOT) { }
-};
+{
+ GemmMethod::GEMM_NATIVE,
+ "native_u8u32_dot_4VLx4",
+ [](const GemmArgs<uint32_t> &args) { return (args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB); },
+ [](const GemmArgs<uint32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); },
+ [](const GemmArgs<uint32_t> &args) { return new GemmNative<native_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "interleaved_u8u32_dot_3VLx8",
+ [](const GemmArgs<uint32_t> &args) { return (args._Ksize>4); },
+ nullptr,
+ [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<interleaved_u8u32_dot_3VLx8, uint8_t, uint32_t>(args); }
+},
#endif
-
-class GemmImpl_gemm_u8_interleaved : public GemmImplementation<uint8_t, uint32_t> {
-public:
- UniqueGemmCommon<uint8_t, uint32_t> instantiate(const GemmArgs<uint32_t> &args) override {
- return UniqueGemmCommon<uint8_t, uint32_t>(new GemmInterleaved<gemm_u8_4x4, uint8_t, uint32_t>(args));
- }
-
- GemmImpl_gemm_u8_interleaved() : GemmImplementation<uint8_t, uint32_t>(GemmMethod::GEMM_INTERLEAVED) { }
-};
-
-static GemmImpl_gemm_u8_interleaved_dot gemm_u8_interleaved_dot_impl{};
-static GemmImpl_gemm_u8_interleaved gemm_u8_interleaved_impl{};
-
-static std::vector<GemmImplementation<uint8_t, uint32_t> *> gemm_u8_methods = {
- &gemm_u8_interleaved_dot_impl,
- &gemm_u8_interleaved_impl
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "gemm_u8_12x8",
+ [](const GemmArgs<uint32_t> &args) { return args._ci->has_dotprod(); },
+ nullptr,
+ [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<gemm_u8_12x8, uint8_t, uint32_t>(args); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "gemm_u8_4x4",
+ nullptr,
+ nullptr,
+ [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<gemm_u8_4x4, uint8_t, uint32_t>(args); }
+},
+{
+ GemmMethod::DEFAULT,
+ "",
+ nullptr,
+ nullptr,
+ nullptr
+}
};
template<>
-std::vector<GemmImplementation<uint8_t, uint32_t> *> &gemm_implementation_list<uint8_t, uint32_t>() {
+const GemmImplementation<uint8_t, uint32_t> *gemm_implementation_list<uint8_t, uint32_t>() {
return gemm_u8_methods;
}
/* Explicitly instantiate the external functions for these types. */
-template UniqueGemmCommon<uint8_t, uint32_t> gemm<uint8_t, uint32_t>(GemmArgs<uint32_t> &args, GemmConfig *cfg);
-template GemmMethod get_gemm_method<uint8_t, uint32_t>(GemmArgs<uint32_t> &args);
-template bool method_is_compatible<uint8_t, uint32_t>(GemmMethod method, GemmArgs<uint32_t> &args);
+template UniqueGemmCommon<uint8_t, uint32_t> gemm<uint8_t, uint32_t>(const GemmArgs<uint32_t> &args);
+template KernelDescription get_gemm_method<uint8_t, uint32_t>(const GemmArgs<uint32_t> &args);
+template bool method_is_compatible<uint8_t, uint32_t>(GemmMethod method, const GemmArgs<uint32_t> &args);
+template std::vector<std::string> get_compatible_kernels<uint8_t, uint32_t> (const GemmArgs<uint32_t> &args);
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp b/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp
index d65971e..40f7f2b 100644
--- a/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -41,7 +41,7 @@
GemmArgs<Tr> newargs = args;
newargs._Msize = args._nbatches;
newargs._nbatches = 1;
- _subgemm = gemm<To,Tr>(newargs, nullptr);
+ _subgemm = gemm<To,Tr>(newargs);
}
void set_arrays(const To *A, const int lda, const int A_batch_stride, const int A_multi_stride,
diff --git a/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp b/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp
index 241c5fe..5cf4276 100644
--- a/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -64,10 +64,11 @@
GemvNativeTransposed(GemvNativeTransposed &) = delete;
GemvNativeTransposed & operator= (GemvNativeTransposed &) = delete;
- GemvNativeTransposed(const CPUInfo *ci, const unsigned int N, const unsigned int K, const unsigned int nmultis, const Tr beta) : _Nsize(N), _Ksize(K), _nmultis(nmultis), _beta(beta), _ci(ci) {
+ GemvNativeTransposed(const GemmArgs<Tr> &args)
+ : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _beta(args._beta), _ci(args._ci) {
/* For now don't do any blocking. TODO: figure out if we should. */
- m_block = K;
- n_block = N;
+ m_block = _Ksize;
+ n_block = _Nsize;
}
// Window is number of out_width blocks times number of multis.
diff --git a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp
index e53ddb2..842339e 100644
--- a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -68,12 +68,21 @@
GemvPretransposed(GemvPretransposed &) = delete;
GemvPretransposed & operator= (GemvPretransposed &) = delete;
- GemvPretransposed(const CPUInfo *ci, const unsigned int N, const unsigned int K, const unsigned int nmultis, const bool trB, const Tr beta) :
- _Nsize(N), _Ksize(K), _nmultis(nmultis), _trB(trB), _beta(beta), _ci(ci),
- _buffer_per_multi(_Ksize * iceildiv(_Nsize, strategy::A_interleave) * strategy::A_interleave) {
+ GemvPretransposed(const GemmArgs<Tr> &args)
+ : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _trB(args._trB), _beta(args._beta), _ci(args._ci),
+ _buffer_per_multi(_Ksize * iceildiv(_Nsize, strategy::A_interleave) * strategy::A_interleave) {
/* For now don't do any blocking. TODO: figure out if we should. */
- m_block = K;
- n_block = N;
+ if (args._cfg && args._cfg->inner_block_size) {
+ m_block = args._cfg->inner_block_size;
+ } else {
+ m_block = _Ksize;
+ }
+
+ if (args._cfg && args._cfg->outer_block_size) {
+ n_block = args._cfg->outer_block_size;
+ } else {
+ n_block = _Nsize;
+ }
}
// Window is number of out_width blocks, times number of multis.
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/generic.cpp
index 418a375..4ad38cb 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -32,9 +32,9 @@
// Kernel implementation.
//
// Assume that "Apanel" points to a chunk of A blocks (each size 8xK) in read-order.
-// Assume that "Bpanel" points to a chunk of B blocks (each size 12xK) in read-order.
+// Assume that "Bpanel" points to a chunk of B blocks (each size 24xK) in read-order.
// Assume that "Cpanel" points to a chunk of C output blocks (each size
-// 12x8), the chunks being arranged in a row major fashion.
+// 24x8), the chunks being arranged in a row major fashion.
//
// Note that the intent of this is that either ablocks or bblocks will be 1
// - this construction allows the output loop to proceed in either order.
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp
new file mode 100644
index 0000000..0c387ff
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __aarch64__
+
+namespace arm_gemm {
+
+// Actual kernel implementations
+void a64_sgemm_nativeA_pretransposeB_16x4(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int);
+
+// Native A/Pretranspose B SGEMM "strategy" class.
+//
+// This describes the characteristics of a family of kernels, in terms of
+// the required interleave properties and the output block size.
+//
+// All kernels in the family must share these characteristics. The actual
+// kernel to be used can be chosen at runtime, based on the CPUInfo
+// structure.
+class sgemm_nativeA_pretransposeB_16x4 {
+public:
+ typedef float operand_type;
+ typedef float result_type;
+
+ typedef void (*kern_type)(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int);
+
+ /* Desired data layout for B buffer (used for pretranspose) */
+ static const int B_interleave = 16;
+ static const int B_block = 1;
+ static const bool B_transpose = true;
+
+ /* Kernel blocking parameters */
+ static int out_width() {
+ return 16;
+ }
+
+ static int out_height() {
+ return 4;
+ }
+
+ static int k_unroll() {
+ return 1;
+ }
+
+ StdTransformsFixed<operand_type, result_type, 4, 16> transforms = {};
+
+ // Default to the generic kernel
+ kern_type kernel=a64_sgemm_nativeA_pretransposeB_16x4;
+
+ sgemm_nativeA_pretransposeB_16x4(const CPUInfo *ci) {
+
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp
new file mode 100644
index 0000000..b2516f8
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp
@@ -0,0 +1,970 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+
+#include <arm_neon.h>
+
+namespace arm_gemm {
+
+void a64_sgemm_nativeA_pretransposeB_16x4(const float *A, int lda, const float *B_panel, float *C, int ldc, float beta, unsigned int numrows, unsigned int numcols, unsigned int K) {
+ const bool oddk = ((K % 8) >= 4);
+ const bool beta0 = (beta == 0.0f);
+ const unsigned int oddones = (K % 4);
+
+ /* Use some small temporary arrays to cope with "ragged" M/N sizes.
+ *
+ * "dummy_A_buf" is used to avoid overreading the A input for ragged M,
+ * and also for output if N is not ragged.
+ *
+ * Since the B input is pretransposed it will be padded as needed, so no
+ * need to worry about overreading that.
+ *
+ * "C_buf" is used to avoid overreading or overwriting the output for
+ * ragged N cases.
+ */
+ float dummy_A_buf[16];
+ float C_buf[64];
+
+ std::memset(dummy_A_buf, 0, sizeof(dummy_A_buf));
+ std::memset(C_buf, 0, sizeof(C_buf));
+
+ for (unsigned int y=0; y<numrows; y+=4) {
+ const float *b_ptr = B_panel;
+ const unsigned int active_rows = std::min(numrows - y, 4U);
+
+ /* Increment values to be used to advance A pointers - these get set
+ * to zero when the corresponding row isn't being used due to ragged
+ * M, so it will just read the dummy buffer repeatedly. Values are
+ * in bytes (8x sizeof(float)). */
+ const unsigned long a_incr1 = (active_rows > 1) ? 32 : 0;
+ const unsigned long a_incr2 = (active_rows > 2) ? 32 : 0;
+ const unsigned long a_incr3 = (active_rows > 3) ? 32 : 0;
+
+ /* Starting points for A pointers on this loop */
+ const float * const a_ptr0_base = A + (y * lda);
+ const float * const a_ptr1_base = (active_rows > 1) ? (a_ptr0_base + lda) : dummy_A_buf;
+ const float * const a_ptr2_base = (active_rows > 2) ? (a_ptr1_base + lda) : dummy_A_buf;
+ const float * const a_ptr3_base = (active_rows > 3) ? (a_ptr2_base + lda) : dummy_A_buf;
+
+ /* Starting points for C pointers on this loop */
+ float *c_ptr0 = C + (y * ldc);
+ float *c_ptr1 = (active_rows > 1) ? (c_ptr0 + ldc) : dummy_A_buf;
+ float *c_ptr2 = (active_rows > 2) ? (c_ptr1 + ldc) : dummy_A_buf;
+ float *c_ptr3 = (active_rows > 3) ? (c_ptr2 + ldc) : dummy_A_buf;
+
+ for (unsigned int x0=0; x0<numcols; x0+=16) {
+ const unsigned int active_cols = std::min(numcols - x0, 16U);
+ const bool use_result_buf = (active_cols < 16);
+
+ /* Reset the A pointers for this loop. */
+ const float *a_ptr0 = a_ptr0_base;
+ const float *a_ptr1 = a_ptr1_base;
+ const float *a_ptr2 = a_ptr2_base;
+ const float *a_ptr3 = a_ptr3_base;
+
+ /* Override C pointers if the result buffer is in use. */
+ if (use_result_buf) {
+ c_ptr0 = C_buf;
+ c_ptr1 = C_buf + 16;
+ c_ptr2 = C_buf + 32;
+ c_ptr3 = C_buf + 48;
+
+ /* If beta is non-zero, prepopulate the result buffer */
+ if (!beta0) {
+ for (unsigned int row=0; row<active_rows; row++) {
+ for (unsigned int col=0; col<active_cols; col++) {
+ C_buf[row * 16 + col] = C[((y + row) * ldc) + (x0 + col)];
+ }
+ }
+ }
+ }
+
+ unsigned int loops = ((K+4)/8) - 1;
+ unsigned int odds = oddones;
+
+ __asm __volatile (
+ "a0 .req v0\n"
+ "a1 .req v1\n"
+ "a2 .req v2\n"
+ "a3 .req v3\n"
+ "a0a .req v4\n"
+ "a1a .req v5\n"
+ "a2a .req v6\n"
+ "a3a .req v7\n"
+ "bb0 .req v8\n"
+ "bb1 .req v9\n"
+ "bb2 .req v10\n"
+ "bb3 .req v11\n"
+ "b0a .req v12\n"
+ "b1a .req v13\n"
+ "b2a .req v14\n"
+ "b3a .req v15\n"
+
+ "a0q .req q0\n"
+ "a1q .req q1\n"
+ "a2q .req q2\n"
+ "a3q .req q3\n"
+ "a0aq .req q4\n"
+ "a1aq .req q5\n"
+ "a2aq .req q6\n"
+ "a3aq .req q7\n"
+ "b0q .req q8\n"
+ "b1q .req q9\n"
+ "b2q .req q10\n"
+ "b3q .req q11\n"
+ "b0aq .req q12\n"
+ "b1aq .req q13\n"
+ "b2aq .req q14\n"
+ "b3aq .req q15\n"
+
+ "movi v16.4s, #0x0\n"
+ "ldr a0q, [%[a_ptr0]]\n"
+ "movi v17.4s, #0x0\n"
+ "ldr b0q, [%[b_ptr]]\n"
+ "movi v18.4s, #0x0\n"
+ "ldr b1q, [%[b_ptr], #16]\n"
+ "movi v19.4s, #0x0\n"
+ "ldr b2q, [%[b_ptr], #32]\n"
+ "movi v20.4s, #0x0\n"
+ "ldr b3q, [%[b_ptr], #48]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "movi v21.4s, #0x0\n"
+ "ldr a1q, [%[a_ptr1]]\n"
+ "movi v22.4s, #0x0\n"
+ "ldr a2q, [%[a_ptr2]]\n"
+ "movi v23.4s, #0x0\n"
+ "ldr a3q, [%[a_ptr3]]\n"
+ "movi v24.4s, #0x0\n"
+ "ldr b0aq, [%[b_ptr]]\n"
+ "movi v25.4s, #0x0\n"
+ "ldr b1aq, [%[b_ptr], #16]\n"
+ "movi v26.4s, #0x0\n"
+ "ldr b2aq, [%[b_ptr], #32]\n"
+ "cbz %w[beta0], 5f\n"
+ "movi v27.4s, #0x0\n"
+ ASM_PREFETCH("[%[b_ptr], #0x40]")
+ "movi v28.4s, #0x0\n"
+ ASM_PREFETCH("[%[b_ptr], #0x80]")
+ "movi v29.4s, #0x0\n"
+ ASM_PREFETCH("[%[b_ptr], #0xC0]")
+ "movi v30.4s, #0x0\n"
+ ASM_PREFETCH("[%[b_ptr], #0x100]")
+ "movi v31.4s, #0x0\n"
+ ASM_PREFETCH("[%[b_ptr], #0x140]")
+ ASM_PREFETCH("[%[b_ptr], #0x180]")
+ ASM_PREFETCH("[%[b_ptr], #0x1C0]")
+ ASM_PREFETCH("[%[b_ptr], #0x200]")
+
+ // Skip if no complete loops.
+ "cbz %w[loops], 4f\n"
+ "b 1f\n"
+
+ // If beta is non-zero, need to load and multiply by beta
+ "5:\n"
+ "ld1r {v4.4s}, [%[betaptr]]\n"
+ "ldr q16, [%[c_ptr0]]\n"
+ "ldr q17, [%[c_ptr0], #16]\n"
+ "ldr q18, [%[c_ptr0], #32]\n"
+ "ldr q19, [%[c_ptr0], #48]\n"
+
+ "ldr q20, [%[c_ptr1]]\n"
+ "fmul v16.4s, v16.4s, v4.4s\n"
+ "ldr q21, [%[c_ptr1], #16]\n"
+ "fmul v17.4s, v17.4s, v4.4s\n"
+ "ldr q22, [%[c_ptr1], #32]\n"
+ "fmul v18.4s, v18.4s, v4.4s\n"
+ "ldr q23, [%[c_ptr1], #48]\n"
+ "fmul v19.4s, v19.4s, v4.4s\n"
+
+ "ldr q24, [%[c_ptr2]]\n"
+ "fmul v20.4s, v20.4s, v4.4s\n"
+ "ldr q25, [%[c_ptr2], #16]\n"
+ "fmul v21.4s, v21.4s, v4.4s\n"
+ "ldr q26, [%[c_ptr2], #32]\n"
+ "fmul v22.4s, v22.4s, v4.4s\n"
+ "ldr q27, [%[c_ptr2], #48]\n"
+ "fmul v23.4s, v23.4s, v4.4s\n"
+
+ "ldr q28, [%[c_ptr3]]\n"
+ "fmul v24.4s, v24.4s, v4.4s\n"
+ ASM_PREFETCH("[%[b_ptr], #0x40]")
+ "ldr q29, [%[c_ptr3], #16]\n"
+ "fmul v25.4s, v25.4s, v4.4s\n"
+ ASM_PREFETCH("[%[b_ptr], #0x80]")
+ "ldr q30, [%[c_ptr3], #32]\n"
+ "fmul v26.4s, v26.4s, v4.4s\n"
+ ASM_PREFETCH("[%[b_ptr], #0xC0]")
+ "ldr q31, [%[c_ptr3], #48]\n"
+ "fmul v27.4s, v27.4s, v4.4s\n"
+ ASM_PREFETCH("[%[b_ptr], #0x100]")
+
+ "fmul v28.4s, v28.4s, v4.4s\n"
+ ASM_PREFETCH("[%[b_ptr], #0x140]")
+ "fmul v29.4s, v29.4s, v4.4s\n"
+ ASM_PREFETCH("[%[b_ptr], #0x180]")
+ "fmul v30.4s, v30.4s, v4.4s\n"
+ ASM_PREFETCH("[%[b_ptr], #0x1C0]")
+ "fmul v31.4s, v31.4s, v4.4s\n"
+ ASM_PREFETCH("[%[b_ptr], #0x200]")
+
+ "cbz %w[loops], 4f\n"
+
+ "1:\n"
+ // Unroll 0
+ "fmla v16.4s, bb0.4s, a0.s[0]\n"
+ ASM_PREFETCH("[%[b_ptr], #0x240]")
+ "fmla v20.4s, bb0.4s, a1.s[0]\n"
+ "ldr b3aq, [%[b_ptr], #48]\n"
+ "fmla v24.4s, bb0.4s, a2.s[0]\n"
+ "fmla v28.4s, bb0.4s, a3.s[0]\n"
+ "ldr b0q, [%[b_ptr], #64]\n"
+
+ "fmla v17.4s, bb1.4s, a0.s[0]\n"
+ "fmla v21.4s, bb1.4s, a1.s[0]\n"
+ "ldr a0aq, [%[a_ptr0], #16]\n"
+ "fmla v25.4s, bb1.4s, a2.s[0]\n"
+ "fmla v29.4s, bb1.4s, a3.s[0]\n"
+ "ldr b1q, [%[b_ptr], #80]\n"
+
+ "fmla v18.4s, bb2.4s, a0.s[0]\n"
+ "fmla v22.4s, bb2.4s, a1.s[0]\n"
+ "ldr a1aq, [%[a_ptr1], #16]\n"
+ "fmla v26.4s, bb2.4s, a2.s[0]\n"
+ "fmla v30.4s, bb2.4s, a3.s[0]\n"
+ "ldr b2q, [%[b_ptr], #96]\n"
+
+ "fmla v19.4s, bb3.4s, a0.s[0]\n"
+ "fmla v23.4s, bb3.4s, a1.s[0]\n"
+ "ldr a2aq, [%[a_ptr2], #16]\n"
+ "fmla v27.4s, bb3.4s, a2.s[0]\n"
+ "fmla v31.4s, bb3.4s, a3.s[0]\n"
+ "ldr b3q, [%[b_ptr], #112]\n"
+
+ // Unroll 1
+ "fmla v16.4s, b0a.4s, a0.s[1]\n"
+ ASM_PREFETCH("[%[b_ptr], #0x280]")
+ "fmla v20.4s, b0a.4s, a1.s[1]\n"
+ "ldr a3aq, [%[a_ptr3], #16]\n"
+ "fmla v24.4s, b0a.4s, a2.s[1]\n"
+ "fmla v28.4s, b0a.4s, a3.s[1]\n"
+ "ldr b0aq, [%[b_ptr], #128]\n"
+
+ "fmla v17.4s, b1a.4s, a0.s[1]\n"
+ "fmla v21.4s, b1a.4s, a1.s[1]\n"
+ "subs %w[loops], %w[loops], #1\n"
+ "fmla v25.4s, b1a.4s, a2.s[1]\n"
+ "fmla v29.4s, b1a.4s, a3.s[1]\n"
+ "ldr b1aq, [%[b_ptr], #144]\n"
+
+ "fmla v18.4s, b2a.4s, a0.s[1]\n"
+ "fmla v22.4s, b2a.4s, a1.s[1]\n"
+ "fmla v26.4s, b2a.4s, a2.s[1]\n"
+ "fmla v30.4s, b2a.4s, a3.s[1]\n"
+ "ldr b2aq, [%[b_ptr], #160]\n"
+
+ "fmla v19.4s, b3a.4s, a0.s[1]\n"
+ "fmla v23.4s, b3a.4s, a1.s[1]\n"
+ "fmla v27.4s, b3a.4s, a2.s[1]\n"
+ "fmla v31.4s, b3a.4s, a3.s[1]\n"
+ "ldr b3aq, [%[b_ptr], #176]\n"
+
+ // Unroll 2
+ "fmla v16.4s, bb0.4s, a0.s[2]\n"
+ ASM_PREFETCH("[%[b_ptr], #0x2C0]")
+ "fmla v20.4s, bb0.4s, a1.s[2]\n"
+ "fmla v24.4s, bb0.4s, a2.s[2]\n"
+ "fmla v28.4s, bb0.4s, a3.s[2]\n"
+ "ldr b0q, [%[b_ptr], #192]\n"
+
+ "fmla v17.4s, bb1.4s, a0.s[2]\n"
+ "add %[a_ptr0], %[a_ptr0], #32\n"
+ "fmla v21.4s, bb1.4s, a1.s[2]\n"
+ "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n"
+ "fmla v25.4s, bb1.4s, a2.s[2]\n"
+ "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n"
+ "fmla v29.4s, bb1.4s, a3.s[2]\n"
+ "ldr b1q, [%[b_ptr], #208]\n"
+
+ "fmla v18.4s, bb2.4s, a0.s[2]\n"
+ "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n"
+ "fmla v22.4s, bb2.4s, a1.s[2]\n"
+ ASM_PREFETCH("[%[a_ptr0], #0x40]")
+ "fmla v26.4s, bb2.4s, a2.s[2]\n"
+ "fmla v30.4s, bb2.4s, a3.s[2]\n"
+ "ldr b2q, [%[b_ptr], #224]\n"
+
+ "fmla v19.4s, bb3.4s, a0.s[2]\n"
+ "fmla v23.4s, bb3.4s, a1.s[2]\n"
+ ASM_PREFETCH("[%[a_ptr1], #0x40]")
+ "fmla v27.4s, bb3.4s, a2.s[2]\n"
+ "fmla v31.4s, bb3.4s, a3.s[2]\n"
+ "ldr b3q, [%[b_ptr], #240]\n"
+
+ // Unroll 3
+ "fmla v16.4s, b0a.4s, a0.s[3]\n"
+ "fmla v20.4s, b0a.4s, a1.s[3]\n"
+ "add %[b_ptr], %[b_ptr], #512\n"
+ "fmla v24.4s, b0a.4s, a2.s[3]\n"
+ "fmla v28.4s, b0a.4s, a3.s[3]\n"
+ "ldr b0aq, [%[b_ptr], #-256]\n"
+
+ "fmla v17.4s, b1a.4s, a0.s[3]\n"
+ ASM_PREFETCH("[%[b_ptr], #0x100]")
+ "fmla v21.4s, b1a.4s, a1.s[3]\n"
+ "fmla v25.4s, b1a.4s, a2.s[3]\n"
+ "fmla v29.4s, b1a.4s, a3.s[3]\n"
+ "ldr b1aq, [%[b_ptr], #-240]\n"
+
+ "fmla v18.4s, b2a.4s, a0.s[3]\n"
+ "fmla v22.4s, b2a.4s, a1.s[3]\n"
+ ASM_PREFETCH("[%[a_ptr2], #0x40]")
+ "fmla v26.4s, b2a.4s, a2.s[3]\n"
+ "fmla v30.4s, b2a.4s, a3.s[3]\n"
+ "ldr b2aq, [%[b_ptr], #-224]\n"
+
+ "fmla v19.4s, b3a.4s, a0.s[3]\n"
+ "fmla v23.4s, b3a.4s, a1.s[3]\n"
+ "ldr a0q, [%[a_ptr0]]\n"
+ "fmla v27.4s, b3a.4s, a2.s[3]\n"
+ "fmla v31.4s, b3a.4s, a3.s[3]\n"
+ "ldr b3aq, [%[b_ptr], #-208]\n"
+
+ // Unroll 4
+ "fmla v16.4s, bb0.4s, a0a.s[0]\n"
+ "fmla v20.4s, bb0.4s, a1a.s[0]\n"
+ ASM_PREFETCH("[%[b_ptr], #0x140]")
+ "fmla v24.4s, bb0.4s, a2a.s[0]\n"
+ "fmla v28.4s, bb0.4s, a3a.s[0]\n"
+ "ldr b0q, [%[b_ptr], #-192]\n"
+
+ "fmla v17.4s, bb1.4s, a0a.s[0]\n"
+ "fmla v21.4s, bb1.4s, a1a.s[0]\n"
+ "ldr a1q, [%[a_ptr1]]\n"
+ "fmla v25.4s, bb1.4s, a2a.s[0]\n"
+ "fmla v29.4s, bb1.4s, a3a.s[0]\n"
+ "ldr b1q, [%[b_ptr], #-176]\n"
+
+ "fmla v18.4s, bb2.4s, a0a.s[0]\n"
+ "fmla v22.4s, bb2.4s, a1a.s[0]\n"
+ "ldr a2q, [%[a_ptr2]]\n"
+ "fmla v26.4s, bb2.4s, a2a.s[0]\n"
+ "fmla v30.4s, bb2.4s, a3a.s[0]\n"
+ "ldr b2q, [%[b_ptr], #-160]\n"
+
+ "fmla v19.4s, bb3.4s, a0a.s[0]\n"
+ "fmla v23.4s, bb3.4s, a1a.s[0]\n"
+ "ldr a3q, [%[a_ptr3]]\n"
+ "fmla v27.4s, bb3.4s, a2a.s[0]\n"
+ "fmla v31.4s, bb3.4s, a3a.s[0]\n"
+ "ldr b3q, [%[b_ptr], #-144]\n"
+
+ // Unroll 5
+ "fmla v16.4s, b0a.4s, a0a.s[1]\n"
+ "fmla v20.4s, b0a.4s, a1a.s[1]\n"
+ ASM_PREFETCH("[%[b_ptr], #0x180]")
+ "fmla v24.4s, b0a.4s, a2a.s[1]\n"
+ "fmla v28.4s, b0a.4s, a3a.s[1]\n"
+ "ldr b0aq, [%[b_ptr], #-128]\n"
+
+ "fmla v17.4s, b1a.4s, a0a.s[1]\n"
+ "fmla v21.4s, b1a.4s, a1a.s[1]\n"
+ ASM_PREFETCH("[%[a_ptr3], #0x40]")
+ "fmla v25.4s, b1a.4s, a2a.s[1]\n"
+ "fmla v29.4s, b1a.4s, a3a.s[1]\n"
+ "ldr b1aq, [%[b_ptr], #-112]\n"
+
+ "fmla v18.4s, b2a.4s, a0a.s[1]\n"
+ "fmla v22.4s, b2a.4s, a1a.s[1]\n"
+ "fmla v26.4s, b2a.4s, a2a.s[1]\n"
+ "fmla v30.4s, b2a.4s, a3a.s[1]\n"
+ "ldr b2aq, [%[b_ptr], #-96]\n"
+
+ "fmla v19.4s, b3a.4s, a0a.s[1]\n"
+ "fmla v23.4s, b3a.4s, a1a.s[1]\n"
+ "fmla v27.4s, b3a.4s, a2a.s[1]\n"
+ "fmla v31.4s, b3a.4s, a3a.s[1]\n"
+ "ldr b3aq, [%[b_ptr], #-80]\n"
+
+ // Unroll 6
+ "fmla v16.4s, bb0.4s, a0a.s[2]\n"
+ "fmla v20.4s, bb0.4s, a1a.s[2]\n"
+ ASM_PREFETCH("[%[b_ptr], #0x1C0]")
+ "fmla v24.4s, bb0.4s, a2a.s[2]\n"
+ "fmla v28.4s, bb0.4s, a3a.s[2]\n"
+ "ldr b0q, [%[b_ptr], #-64]\n"
+
+ "fmla v17.4s, bb1.4s, a0a.s[2]\n"
+ "fmla v21.4s, bb1.4s, a1a.s[2]\n"
+ "fmla v25.4s, bb1.4s, a2a.s[2]\n"
+ "fmla v29.4s, bb1.4s, a3a.s[2]\n"
+ "ldr b1q, [%[b_ptr], #-48]\n"
+
+ "fmla v18.4s, bb2.4s, a0a.s[2]\n"
+ "fmla v22.4s, bb2.4s, a1a.s[2]\n"
+ "fmla v26.4s, bb2.4s, a2a.s[2]\n"
+ "fmla v30.4s, bb2.4s, a3a.s[2]\n"
+ "ldr b2q, [%[b_ptr], #-32]\n"
+
+ "fmla v19.4s, bb3.4s, a0a.s[2]\n"
+ "fmla v23.4s, bb3.4s, a1a.s[2]\n"
+ "fmla v27.4s, bb3.4s, a2a.s[2]\n"
+ "fmla v31.4s, bb3.4s, a3a.s[2]\n"
+ "ldr b3q, [%[b_ptr], #-16]\n"
+
+ // Unroll 7
+ "fmla v16.4s, b0a.4s, a0a.s[3]\n"
+ "fmla v20.4s, b0a.4s, a1a.s[3]\n"
+ "fmla v24.4s, b0a.4s, a2a.s[3]\n"
+ "fmla v28.4s, b0a.4s, a3a.s[3]\n"
+ "ldr b0aq, [%[b_ptr]]\n"
+
+ "fmla v17.4s, b1a.4s, a0a.s[3]\n"
+ "fmla v21.4s, b1a.4s, a1a.s[3]\n"
+ ASM_PREFETCH("[%[b_ptr], #0x200]")
+ "fmla v25.4s, b1a.4s, a2a.s[3]\n"
+ "fmla v29.4s, b1a.4s, a3a.s[3]\n"
+ "ldr b1aq, [%[b_ptr], #16]\n"
+
+ "fmla v18.4s, b2a.4s, a0a.s[3]\n"
+ "fmla v22.4s, b2a.4s, a1a.s[3]\n"
+ "fmla v26.4s, b2a.4s, a2a.s[3]\n"
+ "fmla v30.4s, b2a.4s, a3a.s[3]\n"
+ "ldr b2aq, [%[b_ptr], #32]\n"
+
+ "fmla v19.4s, b3a.4s, a0a.s[3]\n"
+ "fmla v23.4s, b3a.4s, a1a.s[3]\n"
+ "fmla v27.4s, b3a.4s, a2a.s[3]\n"
+ "fmla v31.4s, b3a.4s, a3a.s[3]\n"
+ "bne 1b\n"
+
+ // Skip to here
+ "4:\n"
+
+ // Detached final iteration
+ // Unroll 0
+ "fmla v16.4s, bb0.4s, a0.s[0]\n"
+ "fmla v20.4s, bb0.4s, a1.s[0]\n"
+ "ldr b3aq, [%[b_ptr], #48]\n"
+ "fmla v24.4s, bb0.4s, a2.s[0]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v28.4s, bb0.4s, a3.s[0]\n"
+ "ldr b0q, [%[b_ptr]]\n"
+
+ "fmla v17.4s, bb1.4s, a0.s[0]\n"
+ "cbnz %w[oddk], 2f\n" // Deal with odd K before we load a0a
+ "fmla v21.4s, bb1.4s, a1.s[0]\n"
+ "ldr a0aq, [%[a_ptr0], #16]\n"
+ "fmla v25.4s, bb1.4s, a2.s[0]\n"
+ "fmla v29.4s, bb1.4s, a3.s[0]\n"
+ "ldr b1q, [%[b_ptr], #16]\n"
+
+ "fmla v18.4s, bb2.4s, a0.s[0]\n"
+ "fmla v22.4s, bb2.4s, a1.s[0]\n"
+ "ldr a1aq, [%[a_ptr1], #16]\n"
+ "fmla v26.4s, bb2.4s, a2.s[0]\n"
+ "fmla v30.4s, bb2.4s, a3.s[0]\n"
+ "ldr b2q, [%[b_ptr], #32]\n"
+
+ "fmla v19.4s, bb3.4s, a0.s[0]\n"
+ "fmla v23.4s, bb3.4s, a1.s[0]\n"
+ "ldr a2aq, [%[a_ptr2], #16]\n"
+ "fmla v27.4s, bb3.4s, a2.s[0]\n"
+ "fmla v31.4s, bb3.4s, a3.s[0]\n"
+ "ldr b3q, [%[b_ptr], #48]\n"
+
+ // Unroll 1
+ "fmla v16.4s, b0a.4s, a0.s[1]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v20.4s, b0a.4s, a1.s[1]\n"
+ "ldr a3aq, [%[a_ptr3], #16]\n"
+ "fmla v24.4s, b0a.4s, a2.s[1]\n"
+ "fmla v28.4s, b0a.4s, a3.s[1]\n"
+ "ldr b0aq, [%[b_ptr]]\n"
+
+ "fmla v17.4s, b1a.4s, a0.s[1]\n"
+ "add %[a_ptr0], %[a_ptr0], #32\n"
+ "fmla v21.4s, b1a.4s, a1.s[1]\n"
+ "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n"
+ "fmla v25.4s, b1a.4s, a2.s[1]\n"
+ "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n"
+ "fmla v29.4s, b1a.4s, a3.s[1]\n"
+ "ldr b1aq, [%[b_ptr], #16]\n"
+
+ "fmla v18.4s, b2a.4s, a0.s[1]\n"
+ "fmla v22.4s, b2a.4s, a1.s[1]\n"
+ "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n"
+ "fmla v26.4s, b2a.4s, a2.s[1]\n"
+ "fmla v30.4s, b2a.4s, a3.s[1]\n"
+ "ldr b2aq, [%[b_ptr], #32]\n"
+
+ "fmla v19.4s, b3a.4s, a0.s[1]\n"
+ "fmla v23.4s, b3a.4s, a1.s[1]\n"
+ "fmla v27.4s, b3a.4s, a2.s[1]\n"
+ "fmla v31.4s, b3a.4s, a3.s[1]\n"
+ "ldr b3aq, [%[b_ptr], #48]\n"
+
+ // Unroll 2
+ "fmla v16.4s, bb0.4s, a0.s[2]\n"
+ "fmla v20.4s, bb0.4s, a1.s[2]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v24.4s, bb0.4s, a2.s[2]\n"
+ "fmla v28.4s, bb0.4s, a3.s[2]\n"
+ "ldr b0q, [%[b_ptr]]\n"
+
+ "fmla v17.4s, bb1.4s, a0.s[2]\n"
+ "fmla v21.4s, bb1.4s, a1.s[2]\n"
+ "fmla v25.4s, bb1.4s, a2.s[2]\n"
+ "fmla v29.4s, bb1.4s, a3.s[2]\n"
+ "ldr b1q, [%[b_ptr], #16]\n"
+
+ "fmla v18.4s, bb2.4s, a0.s[2]\n"
+ "fmla v22.4s, bb2.4s, a1.s[2]\n"
+ "fmla v26.4s, bb2.4s, a2.s[2]\n"
+ "fmla v30.4s, bb2.4s, a3.s[2]\n"
+ "ldr b2q, [%[b_ptr], #32]\n"
+
+ "fmla v19.4s, bb3.4s, a0.s[2]\n"
+ "fmla v23.4s, bb3.4s, a1.s[2]\n"
+ "fmla v27.4s, bb3.4s, a2.s[2]\n"
+ "fmla v31.4s, bb3.4s, a3.s[2]\n"
+ "ldr b3q, [%[b_ptr], #48]\n"
+
+ // Unroll 3
+ "fmla v16.4s, b0a.4s, a0.s[3]\n"
+ "fmla v20.4s, b0a.4s, a1.s[3]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v24.4s, b0a.4s, a2.s[3]\n"
+ "fmla v28.4s, b0a.4s, a3.s[3]\n"
+ "ldr b0aq, [%[b_ptr]]\n"
+
+ "fmla v17.4s, b1a.4s, a0.s[3]\n"
+ "fmla v21.4s, b1a.4s, a1.s[3]\n"
+ "fmla v25.4s, b1a.4s, a2.s[3]\n"
+ "fmla v29.4s, b1a.4s, a3.s[3]\n"
+ "ldr b1aq, [%[b_ptr], #16]\n"
+
+ "fmla v18.4s, b2a.4s, a0.s[3]\n"
+ "fmla v22.4s, b2a.4s, a1.s[3]\n"
+ "fmla v26.4s, b2a.4s, a2.s[3]\n"
+ "fmla v30.4s, b2a.4s, a3.s[3]\n"
+ "ldr b2aq, [%[b_ptr], #32]\n"
+
+ "fmla v19.4s, b3a.4s, a0.s[3]\n"
+ "fmla v23.4s, b3a.4s, a1.s[3]\n"
+ "fmla v27.4s, b3a.4s, a2.s[3]\n"
+ "fmla v31.4s, b3a.4s, a3.s[3]\n"
+ "ldr b3aq, [%[b_ptr], #48]\n"
+
+ // Unroll 4
+ "fmla v16.4s, bb0.4s, a0a.s[0]\n"
+ "fmla v20.4s, bb0.4s, a1a.s[0]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v24.4s, bb0.4s, a2a.s[0]\n"
+ "fmla v28.4s, bb0.4s, a3a.s[0]\n"
+ "ldr b0q, [%[b_ptr]]\n"
+
+ "fmla v17.4s, bb1.4s, a0a.s[0]\n"
+ "fmla v21.4s, bb1.4s, a1a.s[0]\n"
+ "fmla v25.4s, bb1.4s, a2a.s[0]\n"
+ "fmla v29.4s, bb1.4s, a3a.s[0]\n"
+ "ldr b1q, [%[b_ptr], #16]\n"
+
+ "fmla v18.4s, bb2.4s, a0a.s[0]\n"
+ "fmla v22.4s, bb2.4s, a1a.s[0]\n"
+ "fmla v26.4s, bb2.4s, a2a.s[0]\n"
+ "fmla v30.4s, bb2.4s, a3a.s[0]\n"
+ "ldr b2q, [%[b_ptr], #32]\n"
+
+ "fmla v19.4s, bb3.4s, a0a.s[0]\n"
+ "fmla v23.4s, bb3.4s, a1a.s[0]\n"
+ "fmla v27.4s, bb3.4s, a2a.s[0]\n"
+ "fmla v31.4s, bb3.4s, a3a.s[0]\n"
+ "ldr b3q, [%[b_ptr], #48]\n"
+
+ // Unroll 5
+ "fmla v16.4s, b0a.4s, a0a.s[1]\n"
+ "fmla v20.4s, b0a.4s, a1a.s[1]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v24.4s, b0a.4s, a2a.s[1]\n"
+ "fmla v28.4s, b0a.4s, a3a.s[1]\n"
+ "ldr b0aq, [%[b_ptr]]\n"
+
+ "fmla v17.4s, b1a.4s, a0a.s[1]\n"
+ "fmla v21.4s, b1a.4s, a1a.s[1]\n"
+ "fmla v25.4s, b1a.4s, a2a.s[1]\n"
+ "fmla v29.4s, b1a.4s, a3a.s[1]\n"
+ "ldr b1aq, [%[b_ptr], #16]\n"
+
+ "fmla v18.4s, b2a.4s, a0a.s[1]\n"
+ "fmla v22.4s, b2a.4s, a1a.s[1]\n"
+ "fmla v26.4s, b2a.4s, a2a.s[1]\n"
+ "fmla v30.4s, b2a.4s, a3a.s[1]\n"
+ "ldr b2aq, [%[b_ptr], #32]\n"
+
+ "fmla v19.4s, b3a.4s, a0a.s[1]\n"
+ "fmla v23.4s, b3a.4s, a1a.s[1]\n"
+ "fmla v27.4s, b3a.4s, a2a.s[1]\n"
+ "fmla v31.4s, b3a.4s, a3a.s[1]\n"
+ "ldr b3aq, [%[b_ptr], #48]\n"
+
+ // Unroll 6
+ "fmla v16.4s, bb0.4s, a0a.s[2]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v20.4s, bb0.4s, a1a.s[2]\n"
+ ASM_PREFETCH("[%[c_ptr0], #0x40]")
+ "fmla v24.4s, bb0.4s, a2a.s[2]\n"
+ "fmla v28.4s, bb0.4s, a3a.s[2]\n"
+
+ "fmla v17.4s, bb1.4s, a0a.s[2]\n"
+ "fmla v21.4s, bb1.4s, a1a.s[2]\n"
+ ASM_PREFETCH("[%[c_ptr1], #0x40]")
+ "fmla v25.4s, bb1.4s, a2a.s[2]\n"
+ "fmla v29.4s, bb1.4s, a3a.s[2]\n"
+
+ "fmla v18.4s, bb2.4s, a0a.s[2]\n"
+ "fmla v22.4s, bb2.4s, a1a.s[2]\n"
+ ASM_PREFETCH("[%[c_ptr2], #0x40]")
+ "fmla v26.4s, bb2.4s, a2a.s[2]\n"
+ "fmla v30.4s, bb2.4s, a3a.s[2]\n"
+
+ "fmla v19.4s, bb3.4s, a0a.s[2]\n"
+ "fmla v23.4s, bb3.4s, a1a.s[2]\n"
+ ASM_PREFETCH("[%[c_ptr3], #0x40]")
+ "fmla v27.4s, bb3.4s, a2a.s[2]\n"
+ "fmla v31.4s, bb3.4s, a3a.s[2]\n"
+
+ // Unroll 7
+ "fmla v16.4s, b0a.4s, a0a.s[3]\n"
+ "fmla v17.4s, b1a.4s, a0a.s[3]\n"
+ "fmla v18.4s, b2a.4s, a0a.s[3]\n"
+ "fmla v19.4s, b3a.4s, a0a.s[3]\n"
+ "cbnz %w[odds], 6f\n"
+
+ "fmla v20.4s, b0a.4s, a1a.s[3]\n"
+ "str q16, [%[c_ptr0]]\n"
+ "fmla v21.4s, b1a.4s, a1a.s[3]\n"
+ "str q17, [%[c_ptr0], #16]\n"
+ "fmla v22.4s, b2a.4s, a1a.s[3]\n"
+ "str q18, [%[c_ptr0], #32]\n"
+ "fmla v23.4s, b3a.4s, a1a.s[3]\n"
+ "str q19, [%[c_ptr0], #48]\n"
+
+ "fmla v24.4s, b0a.4s, a2a.s[3]\n"
+ "str q20, [%[c_ptr1]]\n"
+ "fmla v25.4s, b1a.4s, a2a.s[3]\n"
+ "str q21, [%[c_ptr1], #16]\n"
+ "fmla v26.4s, b2a.4s, a2a.s[3]\n"
+ "str q22, [%[c_ptr1], #32]\n"
+ "fmla v27.4s, b3a.4s, a2a.s[3]\n"
+ "str q23, [%[c_ptr1], #48]\n"
+
+ "fmla v28.4s, b0a.4s, a3a.s[3]\n"
+ "str q24, [%[c_ptr2]]\n"
+ "fmla v29.4s, b1a.4s, a3a.s[3]\n"
+ "str q25, [%[c_ptr2], #16]\n"
+ "fmla v30.4s, b2a.4s, a3a.s[3]\n"
+ "str q26, [%[c_ptr2], #32]\n"
+ "fmla v31.4s, b3a.4s, a3a.s[3]\n"
+ "str q27, [%[c_ptr2], #48]\n"
+ "b 3f\n"
+
+ // Odd K case: Just do 4 more.
+ "2:\n"
+ "fmla v21.4s, bb1.4s, a1.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #16\n"
+ "fmla v25.4s, bb1.4s, a2.s[0]\n"
+ "add %[a_ptr1], %[a_ptr1], #16\n"
+ "fmla v29.4s, bb1.4s, a3.s[0]\n"
+ "ldr b1q, [%[b_ptr], #16]\n"
+
+ "fmla v18.4s, bb2.4s, a0.s[0]\n"
+ "add %[a_ptr2], %[a_ptr2], #16\n"
+ "fmla v22.4s, bb2.4s, a1.s[0]\n"
+ "add %[a_ptr3], %[a_ptr3], #16\n"
+ "fmla v26.4s, bb2.4s, a2.s[0]\n"
+ "fmla v30.4s, bb2.4s, a3.s[0]\n"
+ "ldr b2q, [%[b_ptr], #32]\n"
+
+ "fmla v19.4s, bb3.4s, a0.s[0]\n"
+ "fmla v23.4s, bb3.4s, a1.s[0]\n"
+ "fmla v27.4s, bb3.4s, a2.s[0]\n"
+ "fmla v31.4s, bb3.4s, a3.s[0]\n"
+ "ldr b3q, [%[b_ptr], #48]\n"
+
+ // Unroll 1
+ "fmla v16.4s, b0a.4s, a0.s[1]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v20.4s, b0a.4s, a1.s[1]\n"
+ "fmla v24.4s, b0a.4s, a2.s[1]\n"
+ "fmla v28.4s, b0a.4s, a3.s[1]\n"
+ "ldr b0aq, [%[b_ptr]]\n"
+
+ "fmla v17.4s, b1a.4s, a0.s[1]\n"
+ "fmla v21.4s, b1a.4s, a1.s[1]\n"
+ "fmla v25.4s, b1a.4s, a2.s[1]\n"
+ "fmla v29.4s, b1a.4s, a3.s[1]\n"
+ "ldr b1aq, [%[b_ptr], #16]\n"
+
+ "fmla v18.4s, b2a.4s, a0.s[1]\n"
+ "fmla v22.4s, b2a.4s, a1.s[1]\n"
+ "fmla v26.4s, b2a.4s, a2.s[1]\n"
+ "fmla v30.4s, b2a.4s, a3.s[1]\n"
+ "ldr b2aq, [%[b_ptr], #32]\n"
+
+ "fmla v19.4s, b3a.4s, a0.s[1]\n"
+ "fmla v23.4s, b3a.4s, a1.s[1]\n"
+ "fmla v27.4s, b3a.4s, a2.s[1]\n"
+ "fmla v31.4s, b3a.4s, a3.s[1]\n"
+ "ldr b3aq, [%[b_ptr], #48]\n"
+
+ // Unroll 2
+ "fmla v16.4s, bb0.4s, a0.s[2]\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v20.4s, bb0.4s, a1.s[2]\n"
+ ASM_PREFETCH("[%[c_ptr0], #0x40]")
+ "fmla v24.4s, bb0.4s, a2.s[2]\n"
+ "fmla v28.4s, bb0.4s, a3.s[2]\n"
+
+ "fmla v17.4s, bb1.4s, a0.s[2]\n"
+ "fmla v21.4s, bb1.4s, a1.s[2]\n"
+ ASM_PREFETCH("[%[c_ptr1], #0x40]")
+ "fmla v25.4s, bb1.4s, a2.s[2]\n"
+ "fmla v29.4s, bb1.4s, a3.s[2]\n"
+
+ "fmla v18.4s, bb2.4s, a0.s[2]\n"
+ "fmla v22.4s, bb2.4s, a1.s[2]\n"
+ ASM_PREFETCH("[%[c_ptr2], #0x40]")
+ "fmla v26.4s, bb2.4s, a2.s[2]\n"
+ "fmla v30.4s, bb2.4s, a3.s[2]\n"
+
+ "fmla v19.4s, bb3.4s, a0.s[2]\n"
+ "fmla v23.4s, bb3.4s, a1.s[2]\n"
+ ASM_PREFETCH("[%[c_ptr3], #0x40]")
+ "fmla v27.4s, bb3.4s, a2.s[2]\n"
+ "fmla v31.4s, bb3.4s, a3.s[2]\n"
+
+ // Unroll 3
+ "fmla v16.4s, b0a.4s, a0.s[3]\n"
+ "fmla v17.4s, b1a.4s, a0.s[3]\n"
+ "fmla v18.4s, b2a.4s, a0.s[3]\n"
+ "fmla v19.4s, b3a.4s, a0.s[3]\n"
+ "cbnz %w[odds], 7f\n"
+
+ "fmla v20.4s, b0a.4s, a1.s[3]\n"
+ "str q16, [%[c_ptr0]]\n"
+ "fmla v21.4s, b1a.4s, a1.s[3]\n"
+ "str q17, [%[c_ptr0], #16]\n"
+ "fmla v22.4s, b2a.4s, a1.s[3]\n"
+ "str q18, [%[c_ptr0], #32]\n"
+ "fmla v23.4s, b3a.4s, a1.s[3]\n"
+ "str q19, [%[c_ptr0], #48]\n"
+
+ "fmla v24.4s, b0a.4s, a2.s[3]\n"
+ "str q20, [%[c_ptr1]]\n"
+ "fmla v25.4s, b1a.4s, a2.s[3]\n"
+ "str q21, [%[c_ptr1], #16]\n"
+ "fmla v26.4s, b2a.4s, a2.s[3]\n"
+ "str q22, [%[c_ptr1], #32]\n"
+ "fmla v27.4s, b3a.4s, a2.s[3]\n"
+ "str q23, [%[c_ptr1], #48]\n"
+
+ "fmla v28.4s, b0a.4s, a3.s[3]\n"
+ "str q24, [%[c_ptr2]]\n"
+ "fmla v29.4s, b1a.4s, a3.s[3]\n"
+ "str q25, [%[c_ptr2], #16]\n"
+ "fmla v30.4s, b2a.4s, a3.s[3]\n"
+ "str q26, [%[c_ptr2], #32]\n"
+ "fmla v31.4s, b3a.4s, a3.s[3]\n"
+ "str q27, [%[c_ptr2], #48]\n"
+ "b 3f\n"
+
+ // "Odd ones" - lead in from even
+ "6:\n"
+ "fmla v20.4s, b0a.4s, a1a.s[3]\n"
+ "fmla v21.4s, b1a.4s, a1a.s[3]\n"
+ "ldr b0q, [%[b_ptr]]\n"
+ "fmla v22.4s, b2a.4s, a1a.s[3]\n"
+ "subs %w[odds], %w[odds], #1\n"
+ "fmla v23.4s, b3a.4s, a1a.s[3]\n"
+ "ldr b1q, [%[b_ptr], #16]\n"
+
+ "fmla v24.4s, b0a.4s, a2a.s[3]\n"
+ "fmla v25.4s, b1a.4s, a2a.s[3]\n"
+ "ldr b2q, [%[b_ptr], #32]\n"
+ "fmla v26.4s, b2a.4s, a2a.s[3]\n"
+ "fmla v27.4s, b3a.4s, a2a.s[3]\n"
+ "ldr b3q, [%[b_ptr], #48]\n"
+
+ "fmla v28.4s, b0a.4s, a3a.s[3]\n"
+ "ld1r {a0.4s}, [%[a_ptr0]], #4\n"
+ "fmla v29.4s, b1a.4s, a3a.s[3]\n"
+ "fmla v30.4s, b2a.4s, a3a.s[3]\n"
+ "ld1r {a1.4s}, [%[a_ptr1]], #4\n"
+ "fmla v31.4s, b3a.4s, a3a.s[3]\n"
+
+ "fmla v16.4s, bb0.4s, a0.4s\n"
+ "beq 9f\n"
+ "b 8f\n"
+
+ // "Odd ones" - lead in from odd
+ "7:\n"
+ "fmla v20.4s, b0a.4s, a1.s[3]\n"
+ "subs %w[odds], %w[odds], #1\n"
+ "fmla v21.4s, b1a.4s, a1.s[3]\n"
+ "ldr b0q, [%[b_ptr]]\n"
+ "fmla v22.4s, b2a.4s, a1.s[3]\n"
+ "fmla v23.4s, b3a.4s, a1.s[3]\n"
+ "ldr b1q, [%[b_ptr], #16]\n"
+
+ "fmla v24.4s, b0a.4s, a2.s[3]\n"
+ "fmla v25.4s, b1a.4s, a2.s[3]\n"
+ "ldr b2q, [%[b_ptr], #32]\n"
+ "fmla v26.4s, b2a.4s, a2.s[3]\n"
+ "fmla v27.4s, b3a.4s, a2.s[3]\n"
+ "ldr b3q, [%[b_ptr], #48]\n"
+
+ "fmla v28.4s, b0a.4s, a3.s[3]\n"
+ "ld1r {a0.4s}, [%[a_ptr0]], #4\n"
+ "fmla v29.4s, b1a.4s, a3.s[3]\n"
+ "fmla v30.4s, b2a.4s, a3.s[3]\n"
+ "ld1r {a1.4s}, [%[a_ptr1]], #4\n"
+ "fmla v31.4s, b3a.4s, a3.s[3]\n"
+
+ "fmla v16.4s, bb0.4s, a0.4s\n"
+ "beq 9f\n"
+
+ // "Odd ones" - loop
+ "8:\n"
+ "fmla v17.4s, bb1.4s, a0.4s\n"
+ "ld1r {a2.4s}, [%[a_ptr2]], #4\n"
+ "fmla v18.4s, bb2.4s, a0.4s\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v19.4s, bb3.4s, a0.4s\n"
+ "ld1r {a3.4s}, [%[a_ptr3]], #4\n"
+
+ "fmla v20.4s, bb0.4s, a1.4s\n"
+ "subs %w[odds], %w[odds], #1\n"
+ "fmla v21.4s, bb1.4s, a1.4s\n"
+ "ld1r {a0.4s}, [%[a_ptr0]], #4\n"
+ "fmla v22.4s, bb2.4s, a1.4s\n"
+ "fmla v23.4s, bb3.4s, a1.4s\n"
+ "ld1r {a1.4s}, [%[a_ptr1]], #4\n"
+
+ "fmla v24.4s, bb0.4s, a2.4s\n"
+ "fmla v28.4s, bb0.4s, a3.4s\n"
+ "ldr b0q, [%[b_ptr]]\n"
+ "fmla v25.4s, bb1.4s, a2.4s\n"
+ "fmla v29.4s, bb1.4s, a3.4s\n"
+ "ldr b1q, [%[b_ptr], #16]\n"
+
+ "fmla v26.4s, bb2.4s, a2.4s\n"
+ "fmla v30.4s, bb2.4s, a3.4s\n"
+ "ldr b2q, [%[b_ptr], #32]\n"
+ "fmla v27.4s, bb3.4s, a2.4s\n"
+ "fmla v31.4s, bb3.4s, a3.4s\n"
+ "ldr b3q, [%[b_ptr], #48]\n"
+ "fmla v16.4s, bb0.4s, a0.4s\n"
+ "bne 8b\n"
+
+ // "Odd ones" - detached final iteration
+ "9:\n"
+ "fmla v17.4s, bb1.4s, a0.4s\n"
+ "ld1r {a2.4s}, [%[a_ptr2]], #4\n"
+ "fmla v18.4s, bb2.4s, a0.4s\n"
+ "add %[b_ptr], %[b_ptr], #64\n"
+ "fmla v19.4s, bb3.4s, a0.4s\n"
+ "ld1r {a3.4s}, [%[a_ptr3]], #4\n"
+
+ "fmla v20.4s, bb0.4s, a1.4s\n"
+ "str q16, [%[c_ptr0]]\n"
+ "fmla v21.4s, bb1.4s, a1.4s\n"
+ "str q17, [%[c_ptr0], #16]\n"
+ "fmla v22.4s, bb2.4s, a1.4s\n"
+ "str q18, [%[c_ptr0], #32]\n"
+ "fmla v23.4s, bb3.4s, a1.4s\n"
+ "str q19, [%[c_ptr0], #48]\n"
+
+ "fmla v24.4s, bb0.4s, a2.4s\n"
+ "str q20, [%[c_ptr1]]\n"
+ "fmla v25.4s, bb1.4s, a2.4s\n"
+ "str q21, [%[c_ptr1], #16]\n"
+ "fmla v26.4s, bb2.4s, a2.4s\n"
+ "str q22, [%[c_ptr1], #32]\n"
+ "fmla v27.4s, bb3.4s, a2.4s\n"
+ "str q23, [%[c_ptr1], #48]\n"
+
+ "fmla v28.4s, bb0.4s, a3.4s\n"
+ "str q24, [%[c_ptr2]]\n"
+ "fmla v29.4s, bb1.4s, a3.4s\n"
+ "str q25, [%[c_ptr2], #16]\n"
+ "fmla v30.4s, bb2.4s, a3.4s\n"
+ "str q26, [%[c_ptr2], #32]\n"
+ "fmla v31.4s, bb3.4s, a3.4s\n"
+ "str q27, [%[c_ptr2], #48]\n"
+
+ "3:\n"
+ "str q28, [%[c_ptr3]]\n"
+ // Increment C pointers for next loop - this looks odd if we
+ // are using the result buffer, but it's OK as using the
+ // result buffer implies there will be no next loop.
+ "add %[c_ptr0], %[c_ptr0], #64\n"
+ "str q29, [%[c_ptr3], #16]\n"
+ "add %[c_ptr1], %[c_ptr1], %[a_incr1], LSL #1\n"
+ "str q30, [%[c_ptr3], #32]\n"
+ "add %[c_ptr2], %[c_ptr2], %[a_incr2], LSL #1\n"
+ "str q31, [%[c_ptr3], #48]\n"
+ "add %[c_ptr3], %[c_ptr3], %[a_incr3], LSL #1\n"
+
+ : [a_ptr0] "+r" (a_ptr0), [a_ptr1] "+r" (a_ptr1), [a_ptr2] "+r" (a_ptr2), [a_ptr3] "+r" (a_ptr3),
+ [b_ptr] "+r" (b_ptr), [loops] "+r" (loops), [odds] "+r" (odds),
+ [c_ptr0] "+r" (c_ptr0), [c_ptr1] "+r" (c_ptr1), [c_ptr2] "+r" (c_ptr2), [c_ptr3] "+r" (c_ptr3)
+ : [oddk] "r" (oddk), [beta0] "r" (beta0), [betaptr] "r" (&beta),
+ [a_incr1] "r" (a_incr1), [a_incr2] "r" (a_incr2), [a_incr3] "r" (a_incr3)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
+ "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
+ "cc", "memory"
+ );
+
+ /* Copy results from result buffer if needed. */
+ if (use_result_buf) {
+ for (unsigned int row=0; row<active_rows; row++) {
+ for (unsigned int col=0; col<active_cols; col++) {
+ C[((y + row) * ldc) + (x0 + col)] = C_buf[row * 16 + col];
+ }
+ }
+ }
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp
new file mode 100644
index 0000000..2b58b11
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+
+#include "../std_transforms_sve.hpp"
+
+namespace arm_gemm
+{
+
+// Actual kernel implementations
+void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, float, int, int, int);
+
+class hybrid_fp32_mla_4VLx4
+{
+public:
+ typedef float operand_type;
+ typedef float result_type;
+
+ typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int);
+
+ /* Kernel blocking parameters */
+ static int out_height()
+ {
+ return 4;
+ }
+
+ static int out_width()
+ {
+ return get_vector_length<float>() * 4;
+ }
+
+ static int k_unroll()
+ {
+ return 1;
+ }
+
+ StdTransformsSVE<operand_type, result_type, 4, 4, 1> transforms = {};
+
+ // Default to the generic kernel
+ kern_type kernel=sve_hybrid_fp32_mla_4VLx4;
+
+ hybrid_fp32_mla_4VLx4(const CPUInfo *ci)
+ {
+
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp
new file mode 100644
index 0000000..b8aa825
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp
@@ -0,0 +1,2005 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include <algorithm>
+
+
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+namespace arm_gemm {
+
+void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) {
+ const long beta0 = (beta == 0.0f);
+ const int K_stride = K;
+ const long loops_count = ((K + 4) / 8) - 1;
+ K -= loops_count * 8;
+ const long regs_count = (K / 4) - 1;
+ K -= (regs_count + 1) * 4;
+ const long leftovers = K;
+
+ for (int y=0; y<M; y+=4) {
+ const float * const a_ptr0_base = A + (y * lda);
+ const unsigned long ldab = lda * sizeof(float);
+
+ float *c_ptr0 = C + (y * ldc);
+ const unsigned long ldcb = ldc * sizeof(float);
+
+ for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) {
+ const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>()));
+ const float *betaptr = β
+ long loops = loops_count;
+ long regs = regs_count;
+ long temp = 0;
+ long blocks = leftovers;
+ const float *a_ptr0 = a_ptr0_base;
+ const float *b_ptr0 = B + (K_stride * x0);
+
+ switch(M-y) {
+ case 1:
+ __asm __volatile (
+ "whilelt p6.s, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.s\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "mov z18.s, #0\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z19.s, #0\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "fmul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "2:\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "cbz %[regs], 5f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "b.eq 6f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "b.eq 6f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "b 6f\n"
+ "5:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "b.eq 6f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "b.eq 6f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "6:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 2:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "c_ptr1 .req X1\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.s\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "mov z19.s, #0\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z20.s, #0\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "mov z21.s, #0\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "mov z22.s, #0\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "mov z23.s, #0\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "fmul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "fmul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "fmul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "fmul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "fmul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "fmul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "2:\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "cbz %[regs], 5f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "b.eq 6f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "b.eq 6f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "b 6f\n"
+ "5:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z5.s, p6/z, [a_ptr1]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "b.eq 6f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "b.eq 6f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "6:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq c_ptr1\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
+ );
+ break;
+ case 3:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "c_ptr1 .req X2\n"
+ "c_ptr2 .req X3\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ptrue p7.s\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "mov z19.s, #0\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "mov z20.s, #0\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z21.s, #0\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "mov z22.s, #0\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "mov z23.s, #0\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "mov z24.s, #0\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "mov z25.s, #0\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "mov z26.s, #0\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "mov z27.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "fmul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "fmul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "fmul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "fmul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z24.s, p0/z, [c_ptr2]\n"
+ "fmul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
+ "fmul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
+ "fmul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
+ "fmul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "fmul z24.s, p7/m, z24.s, z15.s\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "fmul z25.s, p7/m, z25.s, z15.s\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "fmul z26.s, p7/m, z26.s, z15.s\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmul z27.s, p7/m, z27.s, z15.s\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "2:\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z24.s, z12.s, z6.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z25.s, z13.s, z6.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z26.s, z14.s, z6.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "fmla z27.s, z15.s, z6.s[3]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "cbz %[regs], 5f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z24.s, z12.s, z6.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z25.s, z13.s, z6.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z26.s, z14.s, z6.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "fmla z27.s, z15.s, z6.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "b.eq 6f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "b.eq 6f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "b 6f\n"
+ "5:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z5.s, p6/z, [a_ptr1]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z6.s, p6/z, [a_ptr2]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "b.eq 6f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "b.eq 6f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "6:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "st1w z24.s, p0, [c_ptr2]\n"
+ "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
+ "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
+ "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
+ );
+ break;
+ default:
+ case 4:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ptrue p7.s\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "mov z19.s, #0\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "mov z20.s, #0\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "mov z21.s, #0\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z22.s, #0\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "mov z23.s, #0\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "mov z24.s, #0\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "mov z25.s, #0\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "mov z26.s, #0\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "mov z27.s, #0\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "mov z28.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z29.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z30.s, #0\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "mov z31.s, #0\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "fmul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "fmul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "fmul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "fmul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z24.s, p0/z, [c_ptr2]\n"
+ "fmul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
+ "fmul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
+ "fmul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
+ "fmul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1w z28.s, p0/z, [c_ptr3]\n"
+ "fmul z24.s, p7/m, z24.s, z15.s\n"
+ "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
+ "fmul z25.s, p7/m, z25.s, z15.s\n"
+ "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
+ "fmul z26.s, p7/m, z26.s, z15.s\n"
+ "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
+ "fmul z27.s, p7/m, z27.s, z15.s\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "fmul z28.s, p7/m, z28.s, z15.s\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "fmul z29.s, p7/m, z29.s, z15.s\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "fmul z30.s, p7/m, z30.s, z15.s\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmul z31.s, p7/m, z31.s, z15.s\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "2:\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
+ "ld1rqw z7.s, p7/z, [a_ptr3]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla z29.s, z9.s, z3.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z30.s, z10.s, z3.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "fmla z31.s, z11.s, z3.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z28.s, z12.s, z3.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z29.s, z13.s, z3.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z30.s, z14.s, z3.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "fmla z31.s, z15.s, z3.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z28.s, z8.s, z3.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z29.s, z9.s, z3.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z30.s, z10.s, z3.s[2]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z31.s, z11.s, z3.s[2]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z28.s, z12.s, z3.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "fmla z29.s, z13.s, z3.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "fmla z30.s, z14.s, z3.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #-0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z28.s, z8.s, z7.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "fmla z29.s, z9.s, z7.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "fmla z30.s, z10.s, z7.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "fmla z31.s, z11.s, z7.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z28.s, z12.s, z7.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "fmla z29.s, z13.s, z7.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "fmla z30.s, z14.s, z7.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "fmla z31.s, z15.s, z7.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z28.s, z8.s, z7.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z29.s, z9.s, z7.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z30.s, z10.s, z7.s[2]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "fmla z31.s, z11.s, z7.s[2]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z24.s, z12.s, z6.s[3]\n"
+ "fmla z28.s, z12.s, z7.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z25.s, z13.s, z6.s[3]\n"
+ "fmla z29.s, z13.s, z7.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z26.s, z14.s, z6.s[3]\n"
+ "fmla z30.s, z14.s, z7.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "fmla z27.s, z15.s, z6.s[3]\n"
+ "fmla z31.s, z15.s, z7.s[3]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "cbz %[regs], 5f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
+ "ld1rqw z7.s, p7/z, [a_ptr3]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z29.s, z9.s, z3.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "fmla z30.s, z10.s, z3.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "fmla z31.s, z11.s, z3.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z28.s, z12.s, z3.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z29.s, z13.s, z3.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z30.s, z14.s, z3.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "fmla z31.s, z15.s, z3.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z28.s, z8.s, z3.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z29.s, z9.s, z3.s[2]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z30.s, z10.s, z3.s[2]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z31.s, z11.s, z3.s[2]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z28.s, z12.s, z3.s[3]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "fmla z29.s, z13.s, z3.s[3]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "fmla z30.s, z14.s, z3.s[3]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z28.s, z8.s, z7.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "fmla z29.s, z9.s, z7.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "fmla z30.s, z10.s, z7.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "fmla z31.s, z11.s, z7.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z28.s, z12.s, z7.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "fmla z29.s, z13.s, z7.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "fmla z30.s, z14.s, z7.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "fmla z31.s, z15.s, z7.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z28.s, z8.s, z7.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z29.s, z9.s, z7.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z30.s, z10.s, z7.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "fmla z31.s, z11.s, z7.s[2]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z24.s, z12.s, z6.s[3]\n"
+ "fmla z28.s, z12.s, z7.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z25.s, z13.s, z6.s[3]\n"
+ "fmla z29.s, z13.s, z7.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z26.s, z14.s, z6.s[3]\n"
+ "fmla z30.s, z14.s, z7.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "fmla z27.s, z15.s, z6.s[3]\n"
+ "fmla z31.s, z15.s, z7.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z29.s, z9.s, z3.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "fmla z30.s, z10.s, z3.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "fmla z31.s, z11.s, z3.s[0]\n"
+ "b.eq 6f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z28.s, z12.s, z3.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z29.s, z13.s, z3.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z30.s, z14.s, z3.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "fmla z31.s, z15.s, z3.s[1]\n"
+ "b.eq 6f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z28.s, z8.s, z3.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z29.s, z9.s, z3.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z30.s, z10.s, z3.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z31.s, z11.s, z3.s[2]\n"
+ "b 6f\n"
+ "5:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z5.s, p6/z, [a_ptr1]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z6.s, p6/z, [a_ptr2]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
+ "ld1rqw z7.s, p6/z, [a_ptr3]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z29.s, z9.s, z3.s[0]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "fmla z30.s, z10.s, z3.s[0]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "fmla z31.s, z11.s, z3.s[0]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z28.s, z12.s, z3.s[1]\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z29.s, z13.s, z3.s[1]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z30.s, z14.s, z3.s[1]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "fmla z31.s, z15.s, z3.s[1]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z28.s, z8.s, z3.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z29.s, z9.s, z3.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z30.s, z10.s, z3.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z31.s, z11.s, z3.s[2]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z28.s, z12.s, z3.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "fmla z29.s, z13.s, z3.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "fmla z30.s, z14.s, z3.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z28.s, z8.s, z7.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "fmla z29.s, z9.s, z7.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "fmla z30.s, z10.s, z7.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "fmla z31.s, z11.s, z7.s[0]\n"
+ "b.eq 6f\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z28.s, z12.s, z7.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "fmla z29.s, z13.s, z7.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "fmla z30.s, z14.s, z7.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "fmla z31.s, z15.s, z7.s[1]\n"
+ "b.eq 6f\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z28.s, z8.s, z7.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z29.s, z9.s, z7.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z30.s, z10.s, z7.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "fmla z31.s, z11.s, z7.s[2]\n"
+ "6:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "st1w z24.s, p0, [c_ptr2]\n"
+ "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
+ "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
+ "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
+ "st1w z28.s, p0, [c_ptr3]\n"
+ "st1w z29.s, p1, [c_ptr3, #1, MUL VL]\n"
+ "st1w z30.s, p2, [c_ptr3, #2, MUL VL]\n"
+ "st1w z31.s, p3, [c_ptr3, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
+ );
+ break;
+ }
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp
index 3fd738e..9d88b60 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -43,7 +43,7 @@
/* Kernel blocking parameters */
static int out_width()
{
- return svcnth() * 3;
+ return get_vector_length<__fp16>() * 3;
}
static int out_height()
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8/generic.cpp
index 92ec888..517895c 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -48,24 +48,24 @@
"mov z8.h, #0\n"
"ptrue p0.h\n"
"mov z9.h, #0\n"
- "ld1rqh z0.h, p0/z, [%[a_ptr]]\n"
"mov z10.h, #0\n"
- "ld1h z2.h, p0/z, [%[b_ptr]]\n"
"mov z11.h, #0\n"
- "ld1h z3.h, p0/z, [%[b_ptr], #1, MUL VL]\n"
"mov z12.h, #0\n"
- "ld1h z4.h, p0/z, [%[b_ptr], #2, MUL VL]\n"
+ "ld1rqh z0.h, p0/z, [%[a_ptr]]\n"
"mov z13.h, #0\n"
- "ld1h z5.h, p0/z, [%[b_ptr], #3, MUL VL]\n"
+ "ld1h z2.h, p0/z, [%[b_ptr]]\n"
"mov z14.h, #0\n"
- "ld1h z6.h, p0/z, [%[b_ptr], #4, MUL VL]\n"
+ "ld1h z3.h, p0/z, [%[b_ptr], #1, MUL VL]\n"
"mov z15.h, #0\n"
- "add %[a_ptr], %[a_ptr], #0x20\n"
+ "ld1h z4.h, p0/z, [%[b_ptr], #2, MUL VL]\n"
"mov z16.h, #0\n"
- "addvl %[b_ptr], %[b_ptr], #6\n"
+ "ld1h z5.h, p0/z, [%[b_ptr], #3, MUL VL]\n"
"mov z17.h, #0\n"
+ "ld1h z6.h, p0/z, [%[b_ptr], #4, MUL VL]\n"
"mov z18.h, #0\n"
+ "add %[a_ptr], %[a_ptr], #0x20\n"
"mov z19.h, #0\n"
+ "addvl %[b_ptr], %[b_ptr], #6\n"
"mov z20.h, #0\n"
"mov z21.h, #0\n"
"mov z22.h, #0\n"
@@ -199,37 +199,31 @@
"fmla z30.h, z7.h, z1.h[6]\n"
"fmla z31.h, z7.h, z1.h[7]\n"
"fmla z8.h, z2.h, z0.h[0]\n"
- "st1h z8.h, p0, [%[c_ptr]]\n"
"fmla z9.h, z2.h, z0.h[1]\n"
"fmla z10.h, z2.h, z0.h[2]\n"
"fmla z11.h, z2.h, z0.h[3]\n"
"fmla z12.h, z2.h, z0.h[4]\n"
+ "st1h z8.h, p0, [%[c_ptr]]\n"
"fmla z13.h, z2.h, z0.h[5]\n"
"fmla z14.h, z2.h, z0.h[6]\n"
"fmla z15.h, z2.h, z0.h[7]\n"
"fmla z16.h, z3.h, z0.h[0]\n"
- "st1h z16.h, p0, [%[c_ptr], #1, MUL VL]\n"
"fmla z17.h, z3.h, z0.h[1]\n"
"fmla z18.h, z3.h, z0.h[2]\n"
"fmla z19.h, z3.h, z0.h[3]\n"
"fmla z20.h, z3.h, z0.h[4]\n"
+ "st1h z16.h, p0, [%[c_ptr], #1, MUL VL]\n"
"fmla z21.h, z3.h, z0.h[5]\n"
"fmla z22.h, z3.h, z0.h[6]\n"
"fmla z23.h, z3.h, z0.h[7]\n"
"fmla z24.h, z4.h, z0.h[0]\n"
- "st1h z24.h, p0, [%[c_ptr], #2, MUL VL]\n"
"fmla z25.h, z4.h, z0.h[1]\n"
- "st1h z9.h, p0, [%[c_ptr], #3, MUL VL]\n"
"fmla z26.h, z4.h, z0.h[2]\n"
- "st1h z17.h, p0, [%[c_ptr], #4, MUL VL]\n"
"fmla z27.h, z4.h, z0.h[3]\n"
- "st1h z25.h, p0, [%[c_ptr], #5, MUL VL]\n"
"fmla z28.h, z4.h, z0.h[4]\n"
- "st1h z10.h, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1h z24.h, p0, [%[c_ptr], #2, MUL VL]\n"
"fmla z29.h, z4.h, z0.h[5]\n"
- "st1h z18.h, p0, [%[c_ptr], #7, MUL VL]\n"
"fmla z30.h, z4.h, z0.h[6]\n"
- "addvl %[c_ptr], %[c_ptr], #16\n"
"fmla z31.h, z4.h, z0.h[7]\n"
"b 4f\n"
"3:\n"
@@ -260,39 +254,39 @@
"fmla z30.h, z4.h, z0.h[6]\n"
"fmla z31.h, z4.h, z0.h[7]\n"
"fmla z8.h, z5.h, z1.h[0]\n"
- "st1h z8.h, p0, [%[c_ptr]]\n"
"fmla z9.h, z5.h, z1.h[1]\n"
"fmla z10.h, z5.h, z1.h[2]\n"
"fmla z11.h, z5.h, z1.h[3]\n"
"fmla z12.h, z5.h, z1.h[4]\n"
+ "st1h z8.h, p0, [%[c_ptr]]\n"
"fmla z13.h, z5.h, z1.h[5]\n"
"fmla z14.h, z5.h, z1.h[6]\n"
"fmla z15.h, z5.h, z1.h[7]\n"
"fmla z16.h, z6.h, z1.h[0]\n"
- "st1h z16.h, p0, [%[c_ptr], #1, MUL VL]\n"
"fmla z17.h, z6.h, z1.h[1]\n"
"fmla z18.h, z6.h, z1.h[2]\n"
"fmla z19.h, z6.h, z1.h[3]\n"
"fmla z20.h, z6.h, z1.h[4]\n"
+ "st1h z16.h, p0, [%[c_ptr], #1, MUL VL]\n"
"fmla z21.h, z6.h, z1.h[5]\n"
"fmla z22.h, z6.h, z1.h[6]\n"
"fmla z23.h, z6.h, z1.h[7]\n"
"fmla z24.h, z7.h, z1.h[0]\n"
- "st1h z24.h, p0, [%[c_ptr], #2, MUL VL]\n"
"fmla z25.h, z7.h, z1.h[1]\n"
- "st1h z9.h, p0, [%[c_ptr], #3, MUL VL]\n"
"fmla z26.h, z7.h, z1.h[2]\n"
- "st1h z17.h, p0, [%[c_ptr], #4, MUL VL]\n"
"fmla z27.h, z7.h, z1.h[3]\n"
- "st1h z25.h, p0, [%[c_ptr], #5, MUL VL]\n"
"fmla z28.h, z7.h, z1.h[4]\n"
- "st1h z10.h, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1h z24.h, p0, [%[c_ptr], #2, MUL VL]\n"
"fmla z29.h, z7.h, z1.h[5]\n"
- "st1h z18.h, p0, [%[c_ptr], #7, MUL VL]\n"
"fmla z30.h, z7.h, z1.h[6]\n"
- "addvl %[c_ptr], %[c_ptr], #16\n"
"fmla z31.h, z7.h, z1.h[7]\n"
"4:\n"
+ "st1h z9.h, p0, [%[c_ptr], #3, MUL VL]\n"
+ "st1h z17.h, p0, [%[c_ptr], #4, MUL VL]\n"
+ "st1h z25.h, p0, [%[c_ptr], #5, MUL VL]\n"
+ "st1h z10.h, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1h z18.h, p0, [%[c_ptr], #7, MUL VL]\n"
+ "addvl %[c_ptr], %[c_ptr], #16\n"
"st1h z26.h, p0, [%[c_ptr], #-8, MUL VL]\n"
"st1h z11.h, p0, [%[c_ptr], #-7, MUL VL]\n"
"st1h z19.h, p0, [%[c_ptr], #-6, MUL VL]\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp
index b2327f3..2e8f261 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -43,7 +43,7 @@
/* Kernel blocking parameters */
static int out_width()
{
- return svcntw() * 3;
+ return get_vector_length<float>() * 3;
}
static int out_height()
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8/generic.cpp
index bb08fc7..88c9840 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -48,22 +48,22 @@
"mov z8.s, #0\n"
"ptrue p0.s\n"
"mov z9.s, #0\n"
- "ld1rqw z0.s, p0/z, [%[a_ptr]]\n"
"mov z10.s, #0\n"
- "ld1w z4.s, p0/z, [%[b_ptr]]\n"
"mov z11.s, #0\n"
- "ld1rqw z1.s, p0/z, [%[a_ptr], #0x10]\n"
"mov z12.s, #0\n"
- "ld1w z5.s, p0/z, [%[b_ptr], #1, MUL VL]\n"
+ "ld1rqw z0.s, p0/z, [%[a_ptr]]\n"
"mov z13.s, #0\n"
- "ld1rqw z2.s, p0/z, [%[a_ptr], #0x20]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr]]\n"
"mov z14.s, #0\n"
- "add %[a_ptr], %[a_ptr], #0x40\n"
+ "ld1rqw z1.s, p0/z, [%[a_ptr], #0x10]\n"
"mov z15.s, #0\n"
- "addvl %[b_ptr], %[b_ptr], #3\n"
+ "ld1w z5.s, p0/z, [%[b_ptr], #1, MUL VL]\n"
"mov z16.s, #0\n"
+ "ld1rqw z2.s, p0/z, [%[a_ptr], #0x20]\n"
"mov z17.s, #0\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
"mov z18.s, #0\n"
+ "addvl %[b_ptr], %[b_ptr], #3\n"
"mov z19.s, #0\n"
"mov z20.s, #0\n"
"mov z21.s, #0\n"
@@ -204,37 +204,31 @@
"fmla z31.s, z6.s, z3.s[3]\n"
"ld1w z6.s, p0/z, [%[b_ptr], #-1, MUL VL]\n"
"fmla z8.s, z4.s, z0.s[0]\n"
- "st1w z8.s, p0, [%[c_ptr]]\n"
"fmla z9.s, z4.s, z0.s[1]\n"
"fmla z10.s, z4.s, z0.s[2]\n"
"fmla z11.s, z4.s, z0.s[3]\n"
"fmla z20.s, z4.s, z1.s[0]\n"
+ "st1w z8.s, p0, [%[c_ptr]]\n"
"fmla z21.s, z4.s, z1.s[1]\n"
"fmla z22.s, z4.s, z1.s[2]\n"
"fmla z23.s, z4.s, z1.s[3]\n"
"fmla z12.s, z5.s, z0.s[0]\n"
- "st1w z12.s, p0, [%[c_ptr], #1, MUL VL]\n"
"fmla z13.s, z5.s, z0.s[1]\n"
"fmla z14.s, z5.s, z0.s[2]\n"
"fmla z15.s, z5.s, z0.s[3]\n"
"fmla z24.s, z5.s, z1.s[0]\n"
+ "st1w z12.s, p0, [%[c_ptr], #1, MUL VL]\n"
"fmla z25.s, z5.s, z1.s[1]\n"
"fmla z26.s, z5.s, z1.s[2]\n"
"fmla z27.s, z5.s, z1.s[3]\n"
"fmla z16.s, z6.s, z0.s[0]\n"
- "st1w z16.s, p0, [%[c_ptr], #2, MUL VL]\n"
"fmla z17.s, z6.s, z0.s[1]\n"
- "st1w z9.s, p0, [%[c_ptr], #3, MUL VL]\n"
"fmla z18.s, z6.s, z0.s[2]\n"
- "st1w z13.s, p0, [%[c_ptr], #4, MUL VL]\n"
"fmla z19.s, z6.s, z0.s[3]\n"
- "st1w z17.s, p0, [%[c_ptr], #5, MUL VL]\n"
"fmla z28.s, z6.s, z1.s[0]\n"
- "st1w z10.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1w z16.s, p0, [%[c_ptr], #2, MUL VL]\n"
"fmla z29.s, z6.s, z1.s[1]\n"
- "st1w z14.s, p0, [%[c_ptr], #7, MUL VL]\n"
"fmla z30.s, z6.s, z1.s[2]\n"
- "addvl %[c_ptr], %[c_ptr], #16\n"
"fmla z31.s, z6.s, z1.s[3]\n"
"b 4f\n"
"3:\n"
@@ -269,39 +263,39 @@
"fmla z31.s, z6.s, z1.s[3]\n"
"ld1w z6.s, p0/z, [%[b_ptr], #-1, MUL VL]\n"
"fmla z8.s, z4.s, z2.s[0]\n"
- "st1w z8.s, p0, [%[c_ptr]]\n"
"fmla z9.s, z4.s, z2.s[1]\n"
"fmla z10.s, z4.s, z2.s[2]\n"
"fmla z11.s, z4.s, z2.s[3]\n"
"fmla z20.s, z4.s, z3.s[0]\n"
+ "st1w z8.s, p0, [%[c_ptr]]\n"
"fmla z21.s, z4.s, z3.s[1]\n"
"fmla z22.s, z4.s, z3.s[2]\n"
"fmla z23.s, z4.s, z3.s[3]\n"
"fmla z12.s, z5.s, z2.s[0]\n"
- "st1w z12.s, p0, [%[c_ptr], #1, MUL VL]\n"
"fmla z13.s, z5.s, z2.s[1]\n"
"fmla z14.s, z5.s, z2.s[2]\n"
"fmla z15.s, z5.s, z2.s[3]\n"
"fmla z24.s, z5.s, z3.s[0]\n"
+ "st1w z12.s, p0, [%[c_ptr], #1, MUL VL]\n"
"fmla z25.s, z5.s, z3.s[1]\n"
"fmla z26.s, z5.s, z3.s[2]\n"
"fmla z27.s, z5.s, z3.s[3]\n"
"fmla z16.s, z6.s, z2.s[0]\n"
- "st1w z16.s, p0, [%[c_ptr], #2, MUL VL]\n"
"fmla z17.s, z6.s, z2.s[1]\n"
- "st1w z9.s, p0, [%[c_ptr], #3, MUL VL]\n"
"fmla z18.s, z6.s, z2.s[2]\n"
- "st1w z13.s, p0, [%[c_ptr], #4, MUL VL]\n"
"fmla z19.s, z6.s, z2.s[3]\n"
- "st1w z17.s, p0, [%[c_ptr], #5, MUL VL]\n"
"fmla z28.s, z6.s, z3.s[0]\n"
- "st1w z10.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1w z16.s, p0, [%[c_ptr], #2, MUL VL]\n"
"fmla z29.s, z6.s, z3.s[1]\n"
- "st1w z14.s, p0, [%[c_ptr], #7, MUL VL]\n"
"fmla z30.s, z6.s, z3.s[2]\n"
- "addvl %[c_ptr], %[c_ptr], #16\n"
"fmla z31.s, z6.s, z3.s[3]\n"
"4:\n"
+ "st1w z9.s, p0, [%[c_ptr], #3, MUL VL]\n"
+ "st1w z13.s, p0, [%[c_ptr], #4, MUL VL]\n"
+ "st1w z17.s, p0, [%[c_ptr], #5, MUL VL]\n"
+ "st1w z10.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1w z14.s, p0, [%[c_ptr], #7, MUL VL]\n"
+ "addvl %[c_ptr], %[c_ptr], #16\n"
"st1w z18.s, p0, [%[c_ptr], #-8, MUL VL]\n"
"st1w z11.s, p0, [%[c_ptr], #-7, MUL VL]\n"
"st1w z15.s, p0, [%[c_ptr], #-6, MUL VL]\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp
index 91aa567..67154e6 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -43,7 +43,7 @@
/* Kernel blocking parameters */
static int out_width()
{
- return svcntw() * 3;
+ return get_vector_length<int32_t>() * 3;
}
static int out_height()
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8/generic.cpp
index 2e994a1..d679c21 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -49,22 +49,22 @@
"mov z8.s, #0\n"
"ptrue p0.b\n"
"mov z9.s, #0\n"
- "ld1rqb z0.b, p0/z, [%[a_ptr]]\n"
"mov z10.s, #0\n"
- "ld1b z4.b, p0/z, [%[b_ptr]]\n"
"mov z11.s, #0\n"
- "ld1rqb z1.b, p0/z, [%[a_ptr], #0x10]\n"
"mov z12.s, #0\n"
- "ld1b z5.b, p0/z, [%[b_ptr], #1, MUL VL]\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr]]\n"
"mov z13.s, #0\n"
- "ld1rqb z2.b, p0/z, [%[a_ptr], #0x20]\n"
+ "ld1b z4.b, p0/z, [%[b_ptr]]\n"
"mov z14.s, #0\n"
- "add %[a_ptr], %[a_ptr], #0x40\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #0x10]\n"
"mov z15.s, #0\n"
- "addvl %[b_ptr], %[b_ptr], #3\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #1, MUL VL]\n"
"mov z16.s, #0\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #0x20]\n"
"mov z17.s, #0\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
"mov z18.s, #0\n"
+ "addvl %[b_ptr], %[b_ptr], #3\n"
"mov z19.s, #0\n"
"mov z20.s, #0\n"
"mov z21.s, #0\n"
@@ -205,37 +205,31 @@
"sdot z31.s, z6.b, z3.b[3]\n"
"ld1b z6.b, p0/z, [%[b_ptr], #-1, MUL VL]\n"
"sdot z8.s, z4.b, z0.b[0]\n"
- "st1w z8.s, p0, [%[c_ptr]]\n"
"sdot z9.s, z4.b, z0.b[1]\n"
"sdot z10.s, z4.b, z0.b[2]\n"
"sdot z11.s, z4.b, z0.b[3]\n"
"sdot z20.s, z4.b, z1.b[0]\n"
+ "st1w z8.s, p0, [%[c_ptr]]\n"
"sdot z21.s, z4.b, z1.b[1]\n"
"sdot z22.s, z4.b, z1.b[2]\n"
"sdot z23.s, z4.b, z1.b[3]\n"
"sdot z12.s, z5.b, z0.b[0]\n"
- "st1w z12.s, p0, [%[c_ptr], #1, MUL VL]\n"
"sdot z13.s, z5.b, z0.b[1]\n"
"sdot z14.s, z5.b, z0.b[2]\n"
"sdot z15.s, z5.b, z0.b[3]\n"
"sdot z24.s, z5.b, z1.b[0]\n"
+ "st1w z12.s, p0, [%[c_ptr], #1, MUL VL]\n"
"sdot z25.s, z5.b, z1.b[1]\n"
"sdot z26.s, z5.b, z1.b[2]\n"
"sdot z27.s, z5.b, z1.b[3]\n"
"sdot z16.s, z6.b, z0.b[0]\n"
- "st1w z16.s, p0, [%[c_ptr], #2, MUL VL]\n"
"sdot z17.s, z6.b, z0.b[1]\n"
- "st1w z9.s, p0, [%[c_ptr], #3, MUL VL]\n"
"sdot z18.s, z6.b, z0.b[2]\n"
- "st1w z13.s, p0, [%[c_ptr], #4, MUL VL]\n"
"sdot z19.s, z6.b, z0.b[3]\n"
- "st1w z17.s, p0, [%[c_ptr], #5, MUL VL]\n"
"sdot z28.s, z6.b, z1.b[0]\n"
- "st1w z10.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1w z16.s, p0, [%[c_ptr], #2, MUL VL]\n"
"sdot z29.s, z6.b, z1.b[1]\n"
- "st1w z14.s, p0, [%[c_ptr], #7, MUL VL]\n"
"sdot z30.s, z6.b, z1.b[2]\n"
- "addvl %[c_ptr], %[c_ptr], #16\n"
"sdot z31.s, z6.b, z1.b[3]\n"
"b 4f\n"
"3:\n"
@@ -270,39 +264,39 @@
"sdot z31.s, z6.b, z1.b[3]\n"
"ld1b z6.b, p0/z, [%[b_ptr], #-1, MUL VL]\n"
"sdot z8.s, z4.b, z2.b[0]\n"
- "st1w z8.s, p0, [%[c_ptr]]\n"
"sdot z9.s, z4.b, z2.b[1]\n"
"sdot z10.s, z4.b, z2.b[2]\n"
"sdot z11.s, z4.b, z2.b[3]\n"
"sdot z20.s, z4.b, z3.b[0]\n"
+ "st1w z8.s, p0, [%[c_ptr]]\n"
"sdot z21.s, z4.b, z3.b[1]\n"
"sdot z22.s, z4.b, z3.b[2]\n"
"sdot z23.s, z4.b, z3.b[3]\n"
"sdot z12.s, z5.b, z2.b[0]\n"
- "st1w z12.s, p0, [%[c_ptr], #1, MUL VL]\n"
"sdot z13.s, z5.b, z2.b[1]\n"
"sdot z14.s, z5.b, z2.b[2]\n"
"sdot z15.s, z5.b, z2.b[3]\n"
"sdot z24.s, z5.b, z3.b[0]\n"
+ "st1w z12.s, p0, [%[c_ptr], #1, MUL VL]\n"
"sdot z25.s, z5.b, z3.b[1]\n"
"sdot z26.s, z5.b, z3.b[2]\n"
"sdot z27.s, z5.b, z3.b[3]\n"
"sdot z16.s, z6.b, z2.b[0]\n"
- "st1w z16.s, p0, [%[c_ptr], #2, MUL VL]\n"
"sdot z17.s, z6.b, z2.b[1]\n"
- "st1w z9.s, p0, [%[c_ptr], #3, MUL VL]\n"
"sdot z18.s, z6.b, z2.b[2]\n"
- "st1w z13.s, p0, [%[c_ptr], #4, MUL VL]\n"
"sdot z19.s, z6.b, z2.b[3]\n"
- "st1w z17.s, p0, [%[c_ptr], #5, MUL VL]\n"
"sdot z28.s, z6.b, z3.b[0]\n"
- "st1w z10.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1w z16.s, p0, [%[c_ptr], #2, MUL VL]\n"
"sdot z29.s, z6.b, z3.b[1]\n"
- "st1w z14.s, p0, [%[c_ptr], #7, MUL VL]\n"
"sdot z30.s, z6.b, z3.b[2]\n"
- "addvl %[c_ptr], %[c_ptr], #16\n"
"sdot z31.s, z6.b, z3.b[3]\n"
"4:\n"
+ "st1w z9.s, p0, [%[c_ptr], #3, MUL VL]\n"
+ "st1w z13.s, p0, [%[c_ptr], #4, MUL VL]\n"
+ "st1w z17.s, p0, [%[c_ptr], #5, MUL VL]\n"
+ "st1w z10.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1w z14.s, p0, [%[c_ptr], #7, MUL VL]\n"
+ "addvl %[c_ptr], %[c_ptr], #16\n"
"st1w z18.s, p0, [%[c_ptr], #-8, MUL VL]\n"
"st1w z11.s, p0, [%[c_ptr], #-7, MUL VL]\n"
"st1w z15.s, p0, [%[c_ptr], #-6, MUL VL]\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp
index ef457e4..628c5a8 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -43,7 +43,7 @@
/* Kernel blocking parameters */
static int out_width()
{
- return svcntw() * 3;
+ return get_vector_length<uint32_t>() * 3;
}
static int out_height()
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp
new file mode 100644
index 0000000..fcc80d9
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+
+
+namespace arm_gemm
+{
+
+// Actual kernel implementations
+void sve_native_fp32_mla_4VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+
+class native_fp32_mla_4VLx4
+{
+public:
+ typedef float operand_type;
+ typedef float result_type;
+
+ typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+
+ /* Kernel blocking parameters */
+ static int out_height()
+ {
+ return 4;
+ }
+
+ static int out_width()
+ {
+ return get_vector_length<float>() * 4;
+ }
+
+ static int k_unroll()
+ {
+ return 1;
+ }
+
+
+
+ // Default to the generic kernel
+ kern_type kernel=sve_native_fp32_mla_4VLx4;
+
+ native_fp32_mla_4VLx4(const CPUInfo *ci)
+ {
+
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp
new file mode 100644
index 0000000..6e22566
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp
@@ -0,0 +1,2066 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include <algorithm>
+
+
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+namespace arm_gemm {
+
+void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) {
+ const long beta0 = (beta == 0.0f);
+ const long loops_count = ((K + 4) / 8) - 1;
+ K -= loops_count * 8;
+ const long regs_count = (K / 4) - 1;
+ K -= (regs_count + 1) * 4;
+ const long leftovers = K;
+
+ for (int y=0; y<M; y+=4) {
+ const float * const a_ptr0_base = A + (y * lda);
+ const unsigned long ldab = lda * sizeof(float);
+
+ float *c_ptr0 = C + (y * ldc);
+ const unsigned long ldcb = ldc * sizeof(float);
+
+ for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) {
+ const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>()));
+ const float *betaptr = β
+ long loops = loops_count;
+ long regs = regs_count;
+ long temp = 0;
+ long blocks = leftovers;
+ const float *a_ptr0 = a_ptr0_base;
+ const float *b_ptr0 = B + x0;
+ long ldbb = ldb * sizeof(float);
+
+ switch(M-y) {
+ case 1:
+ __asm __volatile (
+ "whilelt p6.s, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.s\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "mov z18.s, #0\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z19.s, #0\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "fmul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "2:\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "cbz %[regs], 5f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "b 6f\n"
+ "5:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "6:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 2:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "c_ptr1 .req X1\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.s\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "mov z19.s, #0\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z20.s, #0\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "mov z21.s, #0\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "mov z22.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z23.s, #0\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "fmul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "fmul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "fmul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "fmul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "fmul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "fmul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "2:\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "cbz %[regs], 5f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "b 6f\n"
+ "5:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z5.s, p6/z, [a_ptr1]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "6:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq c_ptr1\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
+ );
+ break;
+ case 3:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "c_ptr1 .req X2\n"
+ "c_ptr2 .req X3\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ptrue p7.s\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "mov z19.s, #0\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "mov z20.s, #0\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z21.s, #0\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "mov z22.s, #0\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "mov z23.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z24.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z25.s, #0\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "mov z26.s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z27.s, #0\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "fmul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "fmul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "fmul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "fmul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z24.s, p0/z, [c_ptr2]\n"
+ "fmul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
+ "fmul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
+ "fmul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
+ "fmul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "fmul z24.s, p7/m, z24.s, z15.s\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "fmul z25.s, p7/m, z25.s, z15.s\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "fmul z26.s, p7/m, z26.s, z15.s\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmul z27.s, p7/m, z27.s, z15.s\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "2:\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z24.s, z12.s, z6.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z13.s, z6.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z26.s, z14.s, z6.s[3]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "fmla z27.s, z15.s, z6.s[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "cbz %[regs], 5f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z24.s, z12.s, z6.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z25.s, z13.s, z6.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z26.s, z14.s, z6.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "fmla z27.s, z15.s, z6.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "b 6f\n"
+ "5:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z5.s, p6/z, [a_ptr1]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z6.s, p6/z, [a_ptr2]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "6:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "st1w z24.s, p0, [c_ptr2]\n"
+ "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
+ "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
+ "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
+ );
+ break;
+ default:
+ case 4:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ptrue p7.s\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "mov z19.s, #0\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "mov z20.s, #0\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "mov z21.s, #0\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "mov z22.s, #0\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "mov z23.s, #0\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "mov z24.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z25.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "mov z26.s, #0\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "mov z27.s, #0\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "mov z28.s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "fmul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "fmul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "fmul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "fmul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z24.s, p0/z, [c_ptr2]\n"
+ "fmul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
+ "fmul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
+ "fmul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
+ "fmul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1w z28.s, p0/z, [c_ptr3]\n"
+ "fmul z24.s, p7/m, z24.s, z15.s\n"
+ "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
+ "fmul z25.s, p7/m, z25.s, z15.s\n"
+ "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
+ "fmul z26.s, p7/m, z26.s, z15.s\n"
+ "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
+ "fmul z27.s, p7/m, z27.s, z15.s\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "fmul z28.s, p7/m, z28.s, z15.s\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "fmul z29.s, p7/m, z29.s, z15.s\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "fmul z30.s, p7/m, z30.s, z15.s\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmul z31.s, p7/m, z31.s, z15.s\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "2:\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z7.s, p7/z, [a_ptr3]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "fmla z29.s, z9.s, z3.s[0]\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "fmla z30.s, z10.s, z3.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "fmla z31.s, z11.s, z3.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z28.s, z12.s, z3.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z29.s, z13.s, z3.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z30.s, z14.s, z3.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "fmla z31.s, z15.s, z3.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z28.s, z8.s, z3.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z29.s, z9.s, z3.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z30.s, z10.s, z3.s[2]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z31.s, z11.s, z3.s[2]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z28.s, z12.s, z3.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "fmla z29.s, z13.s, z3.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "fmla z30.s, z14.s, z3.s[3]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #-0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z28.s, z8.s, z7.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "fmla z29.s, z9.s, z7.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "fmla z30.s, z10.s, z7.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "fmla z31.s, z11.s, z7.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z28.s, z12.s, z7.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "fmla z29.s, z13.s, z7.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "fmla z30.s, z14.s, z7.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "fmla z31.s, z15.s, z7.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z28.s, z8.s, z7.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z29.s, z9.s, z7.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z30.s, z10.s, z7.s[2]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "fmla z31.s, z11.s, z7.s[2]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z24.s, z12.s, z6.s[3]\n"
+ "fmla z28.s, z12.s, z7.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z25.s, z13.s, z6.s[3]\n"
+ "fmla z29.s, z13.s, z7.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z26.s, z14.s, z6.s[3]\n"
+ "fmla z30.s, z14.s, z7.s[3]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "fmla z27.s, z15.s, z6.s[3]\n"
+ "fmla z31.s, z15.s, z7.s[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "cbz %[regs], 5f\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z5.s, p7/z, [a_ptr1]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
+ "ld1rqw z6.s, p7/z, [a_ptr2]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z7.s, p7/z, [a_ptr3]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z29.s, z9.s, z3.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z30.s, z10.s, z3.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "fmla z31.s, z11.s, z3.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z28.s, z12.s, z3.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z29.s, z13.s, z3.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z30.s, z14.s, z3.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "fmla z31.s, z15.s, z3.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z28.s, z8.s, z3.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z29.s, z9.s, z3.s[2]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z30.s, z10.s, z3.s[2]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z31.s, z11.s, z3.s[2]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z28.s, z12.s, z3.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "fmla z29.s, z13.s, z3.s[3]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "fmla z30.s, z14.s, z3.s[3]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z28.s, z8.s, z7.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "fmla z29.s, z9.s, z7.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "fmla z30.s, z10.s, z7.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "fmla z31.s, z11.s, z7.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z28.s, z12.s, z7.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "fmla z29.s, z13.s, z7.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "fmla z30.s, z14.s, z7.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "fmla z31.s, z15.s, z7.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z28.s, z8.s, z7.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z29.s, z9.s, z7.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z30.s, z10.s, z7.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "fmla z31.s, z11.s, z7.s[2]\n"
+ "fmla z16.s, z12.s, z4.s[3]\n"
+ "fmla z20.s, z12.s, z5.s[3]\n"
+ "fmla z24.s, z12.s, z6.s[3]\n"
+ "fmla z28.s, z12.s, z7.s[3]\n"
+ "fmla z17.s, z13.s, z4.s[3]\n"
+ "fmla z21.s, z13.s, z5.s[3]\n"
+ "fmla z25.s, z13.s, z6.s[3]\n"
+ "fmla z29.s, z13.s, z7.s[3]\n"
+ "fmla z18.s, z14.s, z4.s[3]\n"
+ "fmla z22.s, z14.s, z5.s[3]\n"
+ "fmla z26.s, z14.s, z6.s[3]\n"
+ "fmla z30.s, z14.s, z7.s[3]\n"
+ "fmla z19.s, z15.s, z4.s[3]\n"
+ "fmla z23.s, z15.s, z5.s[3]\n"
+ "fmla z27.s, z15.s, z6.s[3]\n"
+ "fmla z31.s, z15.s, z7.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z29.s, z9.s, z3.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "fmla z30.s, z10.s, z3.s[0]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "fmla z31.s, z11.s, z3.s[0]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z28.s, z12.s, z3.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z29.s, z13.s, z3.s[1]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z30.s, z14.s, z3.s[1]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "fmla z31.s, z15.s, z3.s[1]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z28.s, z8.s, z3.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z29.s, z9.s, z3.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z30.s, z10.s, z3.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z31.s, z11.s, z3.s[2]\n"
+ "b 6f\n"
+ "5:\n"
+ "fmla z16.s, z8.s, z0.s[0]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z20.s, z8.s, z1.s[0]\n"
+ "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
+ "fmla z24.s, z8.s, z2.s[0]\n"
+ "ld1rqw z5.s, p6/z, [a_ptr1]\n"
+ "fmla z28.s, z8.s, z3.s[0]\n"
+ "ld1rqw z6.s, p6/z, [a_ptr2]\n"
+ "fmla z17.s, z9.s, z0.s[0]\n"
+ "ld1rqw z7.s, p6/z, [a_ptr3]\n"
+ "fmla z21.s, z9.s, z1.s[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z25.s, z9.s, z2.s[0]\n"
+ "fmla z29.s, z9.s, z3.s[0]\n"
+ "fmla z18.s, z10.s, z0.s[0]\n"
+ "fmla z22.s, z10.s, z1.s[0]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z26.s, z10.s, z2.s[0]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z30.s, z10.s, z3.s[0]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z11.s, z0.s[0]\n"
+ "fmla z23.s, z11.s, z1.s[0]\n"
+ "fmla z27.s, z11.s, z2.s[0]\n"
+ "fmla z31.s, z11.s, z3.s[0]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z0.s[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "fmla z20.s, z12.s, z1.s[1]\n"
+ "fmla z24.s, z12.s, z2.s[1]\n"
+ "fmla z28.s, z12.s, z3.s[1]\n"
+ "fmla z17.s, z13.s, z0.s[1]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "fmla z21.s, z13.s, z1.s[1]\n"
+ "fmla z25.s, z13.s, z2.s[1]\n"
+ "fmla z29.s, z13.s, z3.s[1]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "fmla z18.s, z14.s, z0.s[1]\n"
+ "fmla z22.s, z14.s, z1.s[1]\n"
+ "fmla z26.s, z14.s, z2.s[1]\n"
+ "fmla z30.s, z14.s, z3.s[1]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "fmla z19.s, z15.s, z0.s[1]\n"
+ "fmla z23.s, z15.s, z1.s[1]\n"
+ "fmla z27.s, z15.s, z2.s[1]\n"
+ "fmla z31.s, z15.s, z3.s[1]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z0.s[2]\n"
+ "fmla z20.s, z8.s, z1.s[2]\n"
+ "fmla z24.s, z8.s, z2.s[2]\n"
+ "fmla z28.s, z8.s, z3.s[2]\n"
+ "fmla z17.s, z9.s, z0.s[2]\n"
+ "fmla z21.s, z9.s, z1.s[2]\n"
+ "fmla z25.s, z9.s, z2.s[2]\n"
+ "fmla z29.s, z9.s, z3.s[2]\n"
+ "fmla z18.s, z10.s, z0.s[2]\n"
+ "fmla z22.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z30.s, z10.s, z3.s[2]\n"
+ "fmla z19.s, z11.s, z0.s[2]\n"
+ "fmla z23.s, z11.s, z1.s[2]\n"
+ "fmla z27.s, z11.s, z2.s[2]\n"
+ "fmla z31.s, z11.s, z3.s[2]\n"
+ "fmla z16.s, z12.s, z0.s[3]\n"
+ "fmla z20.s, z12.s, z1.s[3]\n"
+ "fmla z24.s, z12.s, z2.s[3]\n"
+ "fmla z28.s, z12.s, z3.s[3]\n"
+ "fmla z17.s, z13.s, z0.s[3]\n"
+ "fmla z21.s, z13.s, z1.s[3]\n"
+ "fmla z25.s, z13.s, z2.s[3]\n"
+ "fmla z29.s, z13.s, z3.s[3]\n"
+ "fmla z18.s, z14.s, z0.s[3]\n"
+ "fmla z22.s, z14.s, z1.s[3]\n"
+ "fmla z26.s, z14.s, z2.s[3]\n"
+ "fmla z30.s, z14.s, z3.s[3]\n"
+ "fmla z19.s, z15.s, z0.s[3]\n"
+ "fmla z23.s, z15.s, z1.s[3]\n"
+ "fmla z27.s, z15.s, z2.s[3]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[0]\n"
+ "fmla z20.s, z8.s, z5.s[0]\n"
+ "fmla z24.s, z8.s, z6.s[0]\n"
+ "fmla z28.s, z8.s, z7.s[0]\n"
+ "fmla z17.s, z9.s, z4.s[0]\n"
+ "fmla z21.s, z9.s, z5.s[0]\n"
+ "fmla z25.s, z9.s, z6.s[0]\n"
+ "fmla z29.s, z9.s, z7.s[0]\n"
+ "fmla z18.s, z10.s, z4.s[0]\n"
+ "fmla z22.s, z10.s, z5.s[0]\n"
+ "fmla z26.s, z10.s, z6.s[0]\n"
+ "fmla z30.s, z10.s, z7.s[0]\n"
+ "fmla z19.s, z11.s, z4.s[0]\n"
+ "fmla z23.s, z11.s, z5.s[0]\n"
+ "fmla z27.s, z11.s, z6.s[0]\n"
+ "fmla z31.s, z11.s, z7.s[0]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z12.s, z4.s[1]\n"
+ "fmla z20.s, z12.s, z5.s[1]\n"
+ "fmla z24.s, z12.s, z6.s[1]\n"
+ "fmla z28.s, z12.s, z7.s[1]\n"
+ "fmla z17.s, z13.s, z4.s[1]\n"
+ "fmla z21.s, z13.s, z5.s[1]\n"
+ "fmla z25.s, z13.s, z6.s[1]\n"
+ "fmla z29.s, z13.s, z7.s[1]\n"
+ "fmla z18.s, z14.s, z4.s[1]\n"
+ "fmla z22.s, z14.s, z5.s[1]\n"
+ "fmla z26.s, z14.s, z6.s[1]\n"
+ "fmla z30.s, z14.s, z7.s[1]\n"
+ "fmla z19.s, z15.s, z4.s[1]\n"
+ "fmla z23.s, z15.s, z5.s[1]\n"
+ "fmla z27.s, z15.s, z6.s[1]\n"
+ "fmla z31.s, z15.s, z7.s[1]\n"
+ "b.eq 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n"
+ "fmla z16.s, z8.s, z4.s[2]\n"
+ "fmla z20.s, z8.s, z5.s[2]\n"
+ "fmla z24.s, z8.s, z6.s[2]\n"
+ "fmla z28.s, z8.s, z7.s[2]\n"
+ "fmla z17.s, z9.s, z4.s[2]\n"
+ "fmla z21.s, z9.s, z5.s[2]\n"
+ "fmla z25.s, z9.s, z6.s[2]\n"
+ "fmla z29.s, z9.s, z7.s[2]\n"
+ "fmla z18.s, z10.s, z4.s[2]\n"
+ "fmla z22.s, z10.s, z5.s[2]\n"
+ "fmla z26.s, z10.s, z6.s[2]\n"
+ "fmla z30.s, z10.s, z7.s[2]\n"
+ "fmla z19.s, z11.s, z4.s[2]\n"
+ "fmla z23.s, z11.s, z5.s[2]\n"
+ "fmla z27.s, z11.s, z6.s[2]\n"
+ "fmla z31.s, z11.s, z7.s[2]\n"
+ "6:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "st1w z24.s, p0, [c_ptr2]\n"
+ "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
+ "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
+ "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
+ "st1w z28.s, p0, [c_ptr3]\n"
+ "st1w z29.s, p1, [c_ptr3, #1, MUL VL]\n"
+ "st1w z30.s, p2, [c_ptr3, #2, MUL VL]\n"
+ "st1w z31.s, p3, [c_ptr3, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
+ );
+ break;
+ }
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp
new file mode 100644
index 0000000..f5634e3
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+#include <cstdint>
+
+namespace arm_gemm
+{
+
+// Actual kernel implementations
+void sve_native_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int);
+
+class native_s8s32_dot_4VLx4
+{
+public:
+ typedef int8_t operand_type;
+ typedef int32_t result_type;
+
+ typedef void (*kern_type)(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int);
+
+ /* Kernel blocking parameters */
+ static int out_height()
+ {
+ return 4;
+ }
+
+ static int out_width()
+ {
+ return get_vector_length<int32_t>() * 4;
+ }
+
+ static int k_unroll()
+ {
+ return 4;
+ }
+
+
+
+ // Default to the generic kernel
+ kern_type kernel=sve_native_s8s32_dot_4VLx4;
+
+ native_s8s32_dot_4VLx4(const CPUInfo *ci)
+ {
+
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp
new file mode 100644
index 0000000..9c02d95
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp
@@ -0,0 +1,4632 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include <algorithm>
+
+#include <cstdint>
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+namespace arm_gemm {
+
+void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int ldb, int32_t *C, int ldc, int32_t beta, int M, int N, int K) {
+ const long beta0 = (beta == 0);
+ const long loops_count = ((K + 16) / 32) - 1;
+ K -= loops_count * 32;
+ const long regs_count = (K / 16) - 1;
+ K -= (regs_count + 1) * 16;
+ const long leftovers = K;
+ const long blocks_count = K / 4;
+ const long odds_count = K - (blocks_count * 4);
+
+ for (int y=0; y<M; y+=4) {
+ const int8_t * const a_ptr0_base = A + (y * lda);
+ const unsigned long ldab = lda * sizeof(int8_t);
+
+ int32_t *c_ptr0 = C + (y * ldc);
+ const unsigned long ldcb = ldc * sizeof(int32_t);
+
+ for (int x0=0; x0<N; x0+=(4 * get_vector_length<int32_t>())) {
+ const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<int32_t>()));
+ const int32_t *betaptr = β
+ long loops = loops_count;
+ long regs = regs_count;
+ long temp = 0;
+ long blocks = blocks_count;
+ long odds = odds_count;
+ const int8_t *a_ptr0 = a_ptr0_base;
+ const int8_t *b_ptr0 = B + x0;
+ const int8_t *b_ptr1 = b_ptr0 + ldb;
+ const int8_t *b_ptr2 = b_ptr1 + ldb;
+ const int8_t *b_ptr3 = b_ptr2 + ldb;
+ long ldbb = ldb * sizeof(int8_t) * 4;
+
+ switch(M-y) {
+ case 1:
+ __asm __volatile (
+ "whilelt p6.b, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.b\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z18.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mov z19.s, #0\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "mul z19.s, p7/m, z19.s, z15.s\n"
+ "2:\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "cbz %[regs], 5f\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "b.eq 7f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "b.eq 8f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 10f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "11:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "10:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "12:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "b 9f\n"
+ "8:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 13f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "14:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "13:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "15:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "b 9f\n"
+ "7:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 16f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "17:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "16:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "18:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 19f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 20f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "20:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "19:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "21:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "b 9f\n"
+ "5:\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "cbz %[blocks], 22f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "b.eq 23f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "b.eq 24f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 25f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "26:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "25:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "27:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "b 9f\n"
+ "24:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 28f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "29:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "28:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "30:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "b 9f\n"
+ "23:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 31f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "32:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "31:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "33:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "b 9f\n"
+ "22:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 34f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 35f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "35:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "34:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "36:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "9:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 2:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "c_ptr1 .req X1\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.b\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z19.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mov z20.s, #0\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "mov z21.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z22.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "mov z23.s, #0\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "mul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "mul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "mul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "mul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "2:\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z20.s, z12.b, z5.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z21.s, z13.b, z5.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z14.b, z5.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "sdot z23.s, z15.b, z5.b[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "cbz %[regs], 5f\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z20.s, z12.b, z5.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z21.s, z13.b, z5.b[3]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "sdot z22.s, z14.b, z5.b[3]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "sdot z23.s, z15.b, z5.b[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "b.eq 7f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "b.eq 8f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 10f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "11:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "10:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "12:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "b 9f\n"
+ "8:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 13f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "14:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "13:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "15:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "b 9f\n"
+ "7:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 16f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "17:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "16:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "18:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 19f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 20f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "20:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "19:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "21:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "b 9f\n"
+ "5:\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "cbz %[blocks], 22f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "b.eq 23f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "b.eq 24f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 25f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "26:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "25:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "27:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z20.s, z12.b, z5.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z21.s, z13.b, z5.b[3]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "sdot z22.s, z14.b, z5.b[3]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "sdot z23.s, z15.b, z5.b[3]\n"
+ "b 9f\n"
+ "24:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 28f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "29:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "28:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "30:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "b 9f\n"
+ "23:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 31f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "32:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "31:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "33:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "b 9f\n"
+ "22:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 34f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 35f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "35:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "34:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "36:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "9:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq c_ptr1\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
+ );
+ break;
+ case 3:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "c_ptr1 .req X2\n"
+ "c_ptr2 .req X3\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.b\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z19.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z20.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mov z21.s, #0\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "mov z22.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z23.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "mov z24.s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z25.s, #0\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "mov z26.s, #0\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "mov z27.s, #0\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "mul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "mul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "mul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "mul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z24.s, p0/z, [c_ptr2]\n"
+ "mul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
+ "mul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
+ "mul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
+ "mul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mul z24.s, p7/m, z24.s, z15.s\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mul z25.s, p7/m, z25.s, z15.s\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mul z26.s, p7/m, z26.s, z15.s\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mul z27.s, p7/m, z27.s, z15.s\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "2:\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z24.s, z12.b, z2.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z25.s, z13.b, z2.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z14.b, z2.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
+ "sdot z27.s, z15.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #-0x10]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z24.s, z8.b, z6.b[0]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z25.s, z9.b, z6.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z10.b, z6.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "sdot z27.s, z11.b, z6.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z24.s, z12.b, z6.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z25.s, z13.b, z6.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z14.b, z6.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "sdot z27.s, z15.b, z6.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z24.s, z8.b, z6.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z25.s, z9.b, z6.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z10.b, z6.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "sdot z27.s, z11.b, z6.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z5.b[3]\n"
+ "sdot z24.s, z12.b, z6.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z21.s, z13.b, z5.b[3]\n"
+ "sdot z25.s, z13.b, z6.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z14.b, z5.b[3]\n"
+ "sdot z26.s, z14.b, z6.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "sdot z23.s, z15.b, z5.b[3]\n"
+ "sdot z27.s, z15.b, z6.b[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "cbz %[regs], 5f\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z24.s, z12.b, z2.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z25.s, z13.b, z2.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z14.b, z2.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "sdot z27.s, z15.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z24.s, z8.b, z6.b[0]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z25.s, z9.b, z6.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z10.b, z6.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "sdot z27.s, z11.b, z6.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z24.s, z12.b, z6.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z25.s, z13.b, z6.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "sdot z26.s, z14.b, z6.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "sdot z27.s, z15.b, z6.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z24.s, z8.b, z6.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z25.s, z9.b, z6.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z26.s, z10.b, z6.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "sdot z27.s, z11.b, z6.b[2]\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z20.s, z12.b, z5.b[3]\n"
+ "sdot z24.s, z12.b, z6.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z21.s, z13.b, z5.b[3]\n"
+ "sdot z25.s, z13.b, z6.b[3]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "sdot z22.s, z14.b, z5.b[3]\n"
+ "sdot z26.s, z14.b, z6.b[3]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "sdot z23.s, z15.b, z5.b[3]\n"
+ "sdot z27.s, z15.b, z6.b[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "b.eq 7f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "b.eq 8f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 10f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "11:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "10:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "12:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z24.s, z12.b, z2.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z25.s, z13.b, z2.b[3]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "sdot z26.s, z14.b, z2.b[3]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "sdot z27.s, z15.b, z2.b[3]\n"
+ "b 9f\n"
+ "8:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 13f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "14:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "13:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "15:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "b 9f\n"
+ "7:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 16f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "17:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "16:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "18:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 19f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 20f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "20:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "19:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "21:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "b 9f\n"
+ "5:\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr2]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z24.s, z12.b, z2.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z25.s, z13.b, z2.b[3]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "sdot z26.s, z14.b, z2.b[3]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "sdot z27.s, z15.b, z2.b[3]\n"
+ "cbz %[blocks], 22f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z24.s, z8.b, z6.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z25.s, z9.b, z6.b[0]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "sdot z26.s, z10.b, z6.b[0]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "sdot z27.s, z11.b, z6.b[0]\n"
+ "b.eq 23f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z24.s, z12.b, z6.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z25.s, z13.b, z6.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "sdot z26.s, z14.b, z6.b[1]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "sdot z27.s, z15.b, z6.b[1]\n"
+ "b.eq 24f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z24.s, z8.b, z6.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z25.s, z9.b, z6.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "sdot z26.s, z10.b, z6.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "sdot z27.s, z11.b, z6.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 25f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "26:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "25:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "27:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z20.s, z12.b, z5.b[3]\n"
+ "sdot z24.s, z12.b, z6.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z21.s, z13.b, z5.b[3]\n"
+ "sdot z25.s, z13.b, z6.b[3]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "sdot z22.s, z14.b, z5.b[3]\n"
+ "sdot z26.s, z14.b, z6.b[3]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "sdot z23.s, z15.b, z5.b[3]\n"
+ "sdot z27.s, z15.b, z6.b[3]\n"
+ "b 9f\n"
+ "24:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 28f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "29:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "28:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "30:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z24.s, z8.b, z6.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z25.s, z9.b, z6.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "sdot z26.s, z10.b, z6.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "sdot z27.s, z11.b, z6.b[2]\n"
+ "b 9f\n"
+ "23:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 31f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "32:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "31:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "33:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z24.s, z12.b, z6.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z25.s, z13.b, z6.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "sdot z26.s, z14.b, z6.b[1]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "sdot z27.s, z15.b, z6.b[1]\n"
+ "b 9f\n"
+ "22:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 34f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 35f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "35:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "34:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "36:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z24.s, z8.b, z6.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z25.s, z9.b, z6.b[0]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "sdot z26.s, z10.b, z6.b[0]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "sdot z27.s, z11.b, z6.b[0]\n"
+ "9:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "st1w z24.s, p0, [c_ptr2]\n"
+ "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
+ "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
+ "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
+ );
+ break;
+ default:
+ case 4:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.b\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z19.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z20.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z21.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mov z22.s, #0\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "mov z23.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z24.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "mov z25.s, #0\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "mov z26.s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "mov z27.s, #0\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "mov z28.s, #0\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "mul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "mul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "mul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "mul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z24.s, p0/z, [c_ptr2]\n"
+ "mul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
+ "mul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
+ "mul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
+ "mul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1w z28.s, p0/z, [c_ptr3]\n"
+ "mul z24.s, p7/m, z24.s, z15.s\n"
+ "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
+ "mul z25.s, p7/m, z25.s, z15.s\n"
+ "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
+ "mul z26.s, p7/m, z26.s, z15.s\n"
+ "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
+ "mul z27.s, p7/m, z27.s, z15.s\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mul z28.s, p7/m, z28.s, z15.s\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mul z29.s, p7/m, z29.s, z15.s\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mul z30.s, p7/m, z30.s, z15.s\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mul z31.s, p7/m, z31.s, z15.s\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "2:\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "sdot z28.s, z8.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr3]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z29.s, z9.b, z3.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
+ "sdot z30.s, z10.b, z3.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "sdot z31.s, z11.b, z3.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "sdot z28.s, z12.b, z3.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "sdot z29.s, z13.b, z3.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "sdot z30.s, z14.b, z3.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "sdot z31.s, z15.b, z3.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "sdot z28.s, z8.b, z3.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "sdot z29.s, z9.b, z3.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "sdot z30.s, z10.b, z3.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "sdot z31.s, z11.b, z3.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z24.s, z12.b, z2.b[3]\n"
+ "sdot z28.s, z12.b, z3.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z25.s, z13.b, z2.b[3]\n"
+ "sdot z29.s, z13.b, z3.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "sdot z26.s, z14.b, z2.b[3]\n"
+ "sdot z30.s, z14.b, z3.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
+ "sdot z27.s, z15.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #-0x10]\n"
+ "sdot z31.s, z15.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #-0x10]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z24.s, z8.b, z6.b[0]\n"
+ "sdot z28.s, z8.b, z7.b[0]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z25.s, z9.b, z6.b[0]\n"
+ "sdot z29.s, z9.b, z7.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "sdot z26.s, z10.b, z6.b[0]\n"
+ "sdot z30.s, z10.b, z7.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "sdot z27.s, z11.b, z6.b[0]\n"
+ "sdot z31.s, z11.b, z7.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z24.s, z12.b, z6.b[1]\n"
+ "sdot z28.s, z12.b, z7.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z25.s, z13.b, z6.b[1]\n"
+ "sdot z29.s, z13.b, z7.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "sdot z26.s, z14.b, z6.b[1]\n"
+ "sdot z30.s, z14.b, z7.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "sdot z27.s, z15.b, z6.b[1]\n"
+ "sdot z31.s, z15.b, z7.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z24.s, z8.b, z6.b[2]\n"
+ "sdot z28.s, z8.b, z7.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z25.s, z9.b, z6.b[2]\n"
+ "sdot z29.s, z9.b, z7.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "sdot z26.s, z10.b, z6.b[2]\n"
+ "sdot z30.s, z10.b, z7.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "sdot z27.s, z11.b, z6.b[2]\n"
+ "sdot z31.s, z11.b, z7.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z5.b[3]\n"
+ "sdot z24.s, z12.b, z6.b[3]\n"
+ "sdot z28.s, z12.b, z7.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z21.s, z13.b, z5.b[3]\n"
+ "sdot z25.s, z13.b, z6.b[3]\n"
+ "sdot z29.s, z13.b, z7.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "sdot z22.s, z14.b, z5.b[3]\n"
+ "sdot z26.s, z14.b, z6.b[3]\n"
+ "sdot z30.s, z14.b, z7.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "sdot z23.s, z15.b, z5.b[3]\n"
+ "sdot z27.s, z15.b, z6.b[3]\n"
+ "sdot z31.s, z15.b, z7.b[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "cbz %[regs], 5f\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "sdot z28.s, z8.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr3]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z29.s, z9.b, z3.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "sdot z30.s, z10.b, z3.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "sdot z31.s, z11.b, z3.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "sdot z28.s, z12.b, z3.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "sdot z29.s, z13.b, z3.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "sdot z30.s, z14.b, z3.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "sdot z31.s, z15.b, z3.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "sdot z28.s, z8.b, z3.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "sdot z29.s, z9.b, z3.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "sdot z30.s, z10.b, z3.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "sdot z31.s, z11.b, z3.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z24.s, z12.b, z2.b[3]\n"
+ "sdot z28.s, z12.b, z3.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z25.s, z13.b, z2.b[3]\n"
+ "sdot z29.s, z13.b, z3.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "sdot z26.s, z14.b, z2.b[3]\n"
+ "sdot z30.s, z14.b, z3.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "sdot z27.s, z15.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "sdot z31.s, z15.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z24.s, z8.b, z6.b[0]\n"
+ "sdot z28.s, z8.b, z7.b[0]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z25.s, z9.b, z6.b[0]\n"
+ "sdot z29.s, z9.b, z7.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "sdot z26.s, z10.b, z6.b[0]\n"
+ "sdot z30.s, z10.b, z7.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "sdot z27.s, z11.b, z6.b[0]\n"
+ "sdot z31.s, z11.b, z7.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z24.s, z12.b, z6.b[1]\n"
+ "sdot z28.s, z12.b, z7.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z25.s, z13.b, z6.b[1]\n"
+ "sdot z29.s, z13.b, z7.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "sdot z26.s, z14.b, z6.b[1]\n"
+ "sdot z30.s, z14.b, z7.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "sdot z27.s, z15.b, z6.b[1]\n"
+ "sdot z31.s, z15.b, z7.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z24.s, z8.b, z6.b[2]\n"
+ "sdot z28.s, z8.b, z7.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z25.s, z9.b, z6.b[2]\n"
+ "sdot z29.s, z9.b, z7.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "sdot z26.s, z10.b, z6.b[2]\n"
+ "sdot z30.s, z10.b, z7.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "sdot z27.s, z11.b, z6.b[2]\n"
+ "sdot z31.s, z11.b, z7.b[2]\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z20.s, z12.b, z5.b[3]\n"
+ "sdot z24.s, z12.b, z6.b[3]\n"
+ "sdot z28.s, z12.b, z7.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z21.s, z13.b, z5.b[3]\n"
+ "sdot z25.s, z13.b, z6.b[3]\n"
+ "sdot z29.s, z13.b, z7.b[3]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "sdot z22.s, z14.b, z5.b[3]\n"
+ "sdot z26.s, z14.b, z6.b[3]\n"
+ "sdot z30.s, z14.b, z7.b[3]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "sdot z23.s, z15.b, z5.b[3]\n"
+ "sdot z27.s, z15.b, z6.b[3]\n"
+ "sdot z31.s, z15.b, z7.b[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "sdot z28.s, z8.b, z3.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "sdot z29.s, z9.b, z3.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "sdot z30.s, z10.b, z3.b[0]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "sdot z31.s, z11.b, z3.b[0]\n"
+ "b.eq 7f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "sdot z28.s, z12.b, z3.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "sdot z29.s, z13.b, z3.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "sdot z30.s, z14.b, z3.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "sdot z31.s, z15.b, z3.b[1]\n"
+ "b.eq 8f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "sdot z28.s, z8.b, z3.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "sdot z29.s, z9.b, z3.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "sdot z30.s, z10.b, z3.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "sdot z31.s, z11.b, z3.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 10f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "11:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "10:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "12:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z24.s, z12.b, z2.b[3]\n"
+ "sdot z28.s, z12.b, z3.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z25.s, z13.b, z2.b[3]\n"
+ "sdot z29.s, z13.b, z3.b[3]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "sdot z26.s, z14.b, z2.b[3]\n"
+ "sdot z30.s, z14.b, z3.b[3]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "sdot z27.s, z15.b, z2.b[3]\n"
+ "sdot z31.s, z15.b, z3.b[3]\n"
+ "b 9f\n"
+ "8:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 13f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "14:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "13:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "15:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "sdot z28.s, z8.b, z3.b[2]\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "sdot z29.s, z9.b, z3.b[2]\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "sdot z30.s, z10.b, z3.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "sdot z31.s, z11.b, z3.b[2]\n"
+ "b 9f\n"
+ "7:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 16f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "17:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "16:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "18:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "sdot z28.s, z12.b, z3.b[1]\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "sdot z29.s, z13.b, z3.b[1]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "sdot z30.s, z14.b, z3.b[1]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "sdot z31.s, z15.b, z3.b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 19f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 20f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "20:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "19:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "21:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "sdot z28.s, z8.b, z3.b[0]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "sdot z29.s, z9.b, z3.b[0]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "sdot z30.s, z10.b, z3.b[0]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "sdot z31.s, z11.b, z3.b[0]\n"
+ "b 9f\n"
+ "5:\n"
+ "sdot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "sdot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
+ "sdot z28.s, z8.b, z3.b[0]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr2]\n"
+ "sdot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr3]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z29.s, z9.b, z3.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "sdot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "sdot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "sdot z26.s, z10.b, z2.b[0]\n"
+ "sdot z30.s, z10.b, z3.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "sdot z23.s, z11.b, z1.b[0]\n"
+ "sdot z27.s, z11.b, z2.b[0]\n"
+ "sdot z31.s, z11.b, z3.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "sdot z20.s, z12.b, z1.b[1]\n"
+ "sdot z24.s, z12.b, z2.b[1]\n"
+ "sdot z28.s, z12.b, z3.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "sdot z17.s, z13.b, z0.b[1]\n"
+ "sdot z21.s, z13.b, z1.b[1]\n"
+ "sdot z25.s, z13.b, z2.b[1]\n"
+ "sdot z29.s, z13.b, z3.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "sdot z18.s, z14.b, z0.b[1]\n"
+ "sdot z22.s, z14.b, z1.b[1]\n"
+ "sdot z26.s, z14.b, z2.b[1]\n"
+ "sdot z30.s, z14.b, z3.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "sdot z19.s, z15.b, z0.b[1]\n"
+ "sdot z23.s, z15.b, z1.b[1]\n"
+ "sdot z27.s, z15.b, z2.b[1]\n"
+ "sdot z31.s, z15.b, z3.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "sdot z16.s, z8.b, z0.b[2]\n"
+ "sdot z20.s, z8.b, z1.b[2]\n"
+ "sdot z24.s, z8.b, z2.b[2]\n"
+ "sdot z28.s, z8.b, z3.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "sdot z17.s, z9.b, z0.b[2]\n"
+ "sdot z21.s, z9.b, z1.b[2]\n"
+ "sdot z25.s, z9.b, z2.b[2]\n"
+ "sdot z29.s, z9.b, z3.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z18.s, z10.b, z0.b[2]\n"
+ "sdot z22.s, z10.b, z1.b[2]\n"
+ "sdot z26.s, z10.b, z2.b[2]\n"
+ "sdot z30.s, z10.b, z3.b[2]\n"
+ "sdot z19.s, z11.b, z0.b[2]\n"
+ "sdot z23.s, z11.b, z1.b[2]\n"
+ "sdot z27.s, z11.b, z2.b[2]\n"
+ "sdot z31.s, z11.b, z3.b[2]\n"
+ "sdot z16.s, z12.b, z0.b[3]\n"
+ "sdot z20.s, z12.b, z1.b[3]\n"
+ "sdot z24.s, z12.b, z2.b[3]\n"
+ "sdot z28.s, z12.b, z3.b[3]\n"
+ "sdot z17.s, z13.b, z0.b[3]\n"
+ "sdot z21.s, z13.b, z1.b[3]\n"
+ "sdot z25.s, z13.b, z2.b[3]\n"
+ "sdot z29.s, z13.b, z3.b[3]\n"
+ "sdot z18.s, z14.b, z0.b[3]\n"
+ "sdot z22.s, z14.b, z1.b[3]\n"
+ "sdot z26.s, z14.b, z2.b[3]\n"
+ "sdot z30.s, z14.b, z3.b[3]\n"
+ "sdot z19.s, z15.b, z0.b[3]\n"
+ "sdot z23.s, z15.b, z1.b[3]\n"
+ "sdot z27.s, z15.b, z2.b[3]\n"
+ "sdot z31.s, z15.b, z3.b[3]\n"
+ "cbz %[blocks], 22f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z24.s, z8.b, z6.b[0]\n"
+ "sdot z28.s, z8.b, z7.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z25.s, z9.b, z6.b[0]\n"
+ "sdot z29.s, z9.b, z7.b[0]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "sdot z26.s, z10.b, z6.b[0]\n"
+ "sdot z30.s, z10.b, z7.b[0]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "sdot z27.s, z11.b, z6.b[0]\n"
+ "sdot z31.s, z11.b, z7.b[0]\n"
+ "b.eq 23f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z24.s, z12.b, z6.b[1]\n"
+ "sdot z28.s, z12.b, z7.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z25.s, z13.b, z6.b[1]\n"
+ "sdot z29.s, z13.b, z7.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "sdot z26.s, z14.b, z6.b[1]\n"
+ "sdot z30.s, z14.b, z7.b[1]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "sdot z27.s, z15.b, z6.b[1]\n"
+ "sdot z31.s, z15.b, z7.b[1]\n"
+ "b.eq 24f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z24.s, z8.b, z6.b[2]\n"
+ "sdot z28.s, z8.b, z7.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z25.s, z9.b, z6.b[2]\n"
+ "sdot z29.s, z9.b, z7.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "sdot z26.s, z10.b, z6.b[2]\n"
+ "sdot z30.s, z10.b, z7.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "sdot z27.s, z11.b, z6.b[2]\n"
+ "sdot z31.s, z11.b, z7.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 25f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "26:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "25:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "27:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[3]\n"
+ "sdot z20.s, z12.b, z5.b[3]\n"
+ "sdot z24.s, z12.b, z6.b[3]\n"
+ "sdot z28.s, z12.b, z7.b[3]\n"
+ "sdot z17.s, z13.b, z4.b[3]\n"
+ "sdot z21.s, z13.b, z5.b[3]\n"
+ "sdot z25.s, z13.b, z6.b[3]\n"
+ "sdot z29.s, z13.b, z7.b[3]\n"
+ "sdot z18.s, z14.b, z4.b[3]\n"
+ "sdot z22.s, z14.b, z5.b[3]\n"
+ "sdot z26.s, z14.b, z6.b[3]\n"
+ "sdot z30.s, z14.b, z7.b[3]\n"
+ "sdot z19.s, z15.b, z4.b[3]\n"
+ "sdot z23.s, z15.b, z5.b[3]\n"
+ "sdot z27.s, z15.b, z6.b[3]\n"
+ "sdot z31.s, z15.b, z7.b[3]\n"
+ "b 9f\n"
+ "24:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 28f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "29:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "28:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "30:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[2]\n"
+ "sdot z20.s, z8.b, z5.b[2]\n"
+ "sdot z24.s, z8.b, z6.b[2]\n"
+ "sdot z28.s, z8.b, z7.b[2]\n"
+ "sdot z17.s, z9.b, z4.b[2]\n"
+ "sdot z21.s, z9.b, z5.b[2]\n"
+ "sdot z25.s, z9.b, z6.b[2]\n"
+ "sdot z29.s, z9.b, z7.b[2]\n"
+ "sdot z18.s, z10.b, z4.b[2]\n"
+ "sdot z22.s, z10.b, z5.b[2]\n"
+ "sdot z26.s, z10.b, z6.b[2]\n"
+ "sdot z30.s, z10.b, z7.b[2]\n"
+ "sdot z19.s, z11.b, z4.b[2]\n"
+ "sdot z23.s, z11.b, z5.b[2]\n"
+ "sdot z27.s, z11.b, z6.b[2]\n"
+ "sdot z31.s, z11.b, z7.b[2]\n"
+ "b 9f\n"
+ "23:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 31f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "32:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "31:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "33:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "sdot z16.s, z12.b, z4.b[1]\n"
+ "sdot z20.s, z12.b, z5.b[1]\n"
+ "sdot z24.s, z12.b, z6.b[1]\n"
+ "sdot z28.s, z12.b, z7.b[1]\n"
+ "sdot z17.s, z13.b, z4.b[1]\n"
+ "sdot z21.s, z13.b, z5.b[1]\n"
+ "sdot z25.s, z13.b, z6.b[1]\n"
+ "sdot z29.s, z13.b, z7.b[1]\n"
+ "sdot z18.s, z14.b, z4.b[1]\n"
+ "sdot z22.s, z14.b, z5.b[1]\n"
+ "sdot z26.s, z14.b, z6.b[1]\n"
+ "sdot z30.s, z14.b, z7.b[1]\n"
+ "sdot z19.s, z15.b, z4.b[1]\n"
+ "sdot z23.s, z15.b, z5.b[1]\n"
+ "sdot z27.s, z15.b, z6.b[1]\n"
+ "sdot z31.s, z15.b, z7.b[1]\n"
+ "b 9f\n"
+ "22:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 34f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 35f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "35:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "34:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "36:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "sdot z16.s, z8.b, z4.b[0]\n"
+ "sdot z20.s, z8.b, z5.b[0]\n"
+ "sdot z24.s, z8.b, z6.b[0]\n"
+ "sdot z28.s, z8.b, z7.b[0]\n"
+ "sdot z17.s, z9.b, z4.b[0]\n"
+ "sdot z21.s, z9.b, z5.b[0]\n"
+ "sdot z25.s, z9.b, z6.b[0]\n"
+ "sdot z29.s, z9.b, z7.b[0]\n"
+ "sdot z18.s, z10.b, z4.b[0]\n"
+ "sdot z22.s, z10.b, z5.b[0]\n"
+ "sdot z26.s, z10.b, z6.b[0]\n"
+ "sdot z30.s, z10.b, z7.b[0]\n"
+ "sdot z19.s, z11.b, z4.b[0]\n"
+ "sdot z23.s, z11.b, z5.b[0]\n"
+ "sdot z27.s, z11.b, z6.b[0]\n"
+ "sdot z31.s, z11.b, z7.b[0]\n"
+ "9:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "st1w z24.s, p0, [c_ptr2]\n"
+ "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
+ "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
+ "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
+ "st1w z28.s, p0, [c_ptr3]\n"
+ "st1w z29.s, p1, [c_ptr3, #1, MUL VL]\n"
+ "st1w z30.s, p2, [c_ptr3, #2, MUL VL]\n"
+ "st1w z31.s, p3, [c_ptr3, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
+ );
+ break;
+ }
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp
new file mode 100644
index 0000000..f5ebad8
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+#include <cstdint>
+
+
+namespace arm_gemm
+{
+
+// Actual kernel implementations
+void sve_native_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int);
+
+class native_u8u32_dot_4VLx4
+{
+public:
+ typedef uint8_t operand_type;
+ typedef uint32_t result_type;
+
+ typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int);
+
+ /* Kernel blocking parameters */
+ static int out_height()
+ {
+ return 4;
+ }
+
+ static int out_width()
+ {
+ return get_vector_length<uint32_t>() * 4;
+ }
+
+ static int k_unroll()
+ {
+ return 4;
+ }
+
+
+
+ // Default to the generic kernel
+ kern_type kernel=sve_native_u8u32_dot_4VLx4;
+
+ native_u8u32_dot_4VLx4(const CPUInfo *ci)
+ {
+
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp
new file mode 100644
index 0000000..7d89948
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp
@@ -0,0 +1,4632 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include <algorithm>
+
+#include <cstdint>
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+namespace arm_gemm {
+
+void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ldb, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) {
+ const long beta0 = (beta == 0u);
+ const long loops_count = ((K + 16) / 32) - 1;
+ K -= loops_count * 32;
+ const long regs_count = (K / 16) - 1;
+ K -= (regs_count + 1) * 16;
+ const long leftovers = K;
+ const long blocks_count = K / 4;
+ const long odds_count = K - (blocks_count * 4);
+
+ for (int y=0; y<M; y+=4) {
+ const uint8_t * const a_ptr0_base = A + (y * lda);
+ const unsigned long ldab = lda * sizeof(uint8_t);
+
+ uint32_t *c_ptr0 = C + (y * ldc);
+ const unsigned long ldcb = ldc * sizeof(uint32_t);
+
+ for (int x0=0; x0<N; x0+=(4 * get_vector_length<uint32_t>())) {
+ const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<uint32_t>()));
+ const uint32_t *betaptr = β
+ long loops = loops_count;
+ long regs = regs_count;
+ long temp = 0;
+ long blocks = blocks_count;
+ long odds = odds_count;
+ const uint8_t *a_ptr0 = a_ptr0_base;
+ const uint8_t *b_ptr0 = B + x0;
+ const uint8_t *b_ptr1 = b_ptr0 + ldb;
+ const uint8_t *b_ptr2 = b_ptr1 + ldb;
+ const uint8_t *b_ptr3 = b_ptr2 + ldb;
+ long ldbb = ldb * sizeof(uint8_t) * 4;
+
+ switch(M-y) {
+ case 1:
+ __asm __volatile (
+ "whilelt p6.b, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.b\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z18.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mov z19.s, #0\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "mul z19.s, p7/m, z19.s, z15.s\n"
+ "2:\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "cbz %[regs], 5f\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "b.eq 7f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "b.eq 8f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 10f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "11:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "10:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "12:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "b 9f\n"
+ "8:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 13f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "14:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "13:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "15:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "b 9f\n"
+ "7:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 16f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "17:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "16:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "18:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 19f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 20f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "20:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "19:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "21:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "b 9f\n"
+ "5:\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "cbz %[blocks], 22f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "b.eq 23f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "b.eq 24f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 25f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "26:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "25:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "27:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "b 9f\n"
+ "24:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 28f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "29:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "28:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "30:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "b 9f\n"
+ "23:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 31f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "32:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "31:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "33:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "b 9f\n"
+ "22:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 34f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 35f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "35:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "34:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "36:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "9:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 2:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "c_ptr1 .req X1\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.b\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z19.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mov z20.s, #0\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "mov z21.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z22.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "mov z23.s, #0\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "mul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "mul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "mul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "mul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "2:\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z20.s, z12.b, z5.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z21.s, z13.b, z5.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z14.b, z5.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "udot z23.s, z15.b, z5.b[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "cbz %[regs], 5f\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z20.s, z12.b, z5.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z21.s, z13.b, z5.b[3]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "udot z22.s, z14.b, z5.b[3]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "udot z23.s, z15.b, z5.b[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "b.eq 7f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "b.eq 8f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 10f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "11:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "10:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "12:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "b 9f\n"
+ "8:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 13f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "14:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "13:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "15:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "b 9f\n"
+ "7:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 16f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "17:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "16:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "18:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 19f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 20f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "20:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "19:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "21:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "b 9f\n"
+ "5:\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "cbz %[blocks], 22f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "b.eq 23f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "b.eq 24f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 25f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "26:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "25:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "27:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z20.s, z12.b, z5.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z21.s, z13.b, z5.b[3]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "udot z22.s, z14.b, z5.b[3]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "udot z23.s, z15.b, z5.b[3]\n"
+ "b 9f\n"
+ "24:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 28f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "29:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "28:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "30:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "b 9f\n"
+ "23:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 31f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "32:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "31:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "33:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "b 9f\n"
+ "22:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 34f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 35f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "35:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "34:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "36:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "9:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq c_ptr1\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
+ );
+ break;
+ case 3:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "c_ptr1 .req X2\n"
+ "c_ptr2 .req X3\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.b\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z19.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z20.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mov z21.s, #0\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "mov z22.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z23.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "mov z24.s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z25.s, #0\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "mov z26.s, #0\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "mov z27.s, #0\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "mul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "mul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "mul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "mul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z24.s, p0/z, [c_ptr2]\n"
+ "mul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
+ "mul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
+ "mul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
+ "mul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mul z24.s, p7/m, z24.s, z15.s\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mul z25.s, p7/m, z25.s, z15.s\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mul z26.s, p7/m, z26.s, z15.s\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mul z27.s, p7/m, z27.s, z15.s\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "2:\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z24.s, z12.b, z2.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z25.s, z13.b, z2.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z14.b, z2.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
+ "udot z27.s, z15.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #-0x10]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z24.s, z8.b, z6.b[0]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z25.s, z9.b, z6.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z10.b, z6.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "udot z27.s, z11.b, z6.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z24.s, z12.b, z6.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z25.s, z13.b, z6.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z14.b, z6.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "udot z27.s, z15.b, z6.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z24.s, z8.b, z6.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z25.s, z9.b, z6.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z10.b, z6.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "udot z27.s, z11.b, z6.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z5.b[3]\n"
+ "udot z24.s, z12.b, z6.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z21.s, z13.b, z5.b[3]\n"
+ "udot z25.s, z13.b, z6.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z14.b, z5.b[3]\n"
+ "udot z26.s, z14.b, z6.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "udot z23.s, z15.b, z5.b[3]\n"
+ "udot z27.s, z15.b, z6.b[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "cbz %[regs], 5f\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z24.s, z12.b, z2.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z25.s, z13.b, z2.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z14.b, z2.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "udot z27.s, z15.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z24.s, z8.b, z6.b[0]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z25.s, z9.b, z6.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z10.b, z6.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "udot z27.s, z11.b, z6.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z24.s, z12.b, z6.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z25.s, z13.b, z6.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "udot z26.s, z14.b, z6.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "udot z27.s, z15.b, z6.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z24.s, z8.b, z6.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z25.s, z9.b, z6.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z26.s, z10.b, z6.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "udot z27.s, z11.b, z6.b[2]\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z20.s, z12.b, z5.b[3]\n"
+ "udot z24.s, z12.b, z6.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z21.s, z13.b, z5.b[3]\n"
+ "udot z25.s, z13.b, z6.b[3]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "udot z22.s, z14.b, z5.b[3]\n"
+ "udot z26.s, z14.b, z6.b[3]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "udot z23.s, z15.b, z5.b[3]\n"
+ "udot z27.s, z15.b, z6.b[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "b.eq 7f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "b.eq 8f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 10f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "11:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "10:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "12:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z24.s, z12.b, z2.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z25.s, z13.b, z2.b[3]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "udot z26.s, z14.b, z2.b[3]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "udot z27.s, z15.b, z2.b[3]\n"
+ "b 9f\n"
+ "8:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 13f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "14:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "13:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "15:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "b 9f\n"
+ "7:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 16f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "17:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "16:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "18:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 19f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 20f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "20:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "19:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "21:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "b 9f\n"
+ "5:\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr2]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z24.s, z12.b, z2.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z25.s, z13.b, z2.b[3]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "udot z26.s, z14.b, z2.b[3]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "udot z27.s, z15.b, z2.b[3]\n"
+ "cbz %[blocks], 22f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z24.s, z8.b, z6.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z25.s, z9.b, z6.b[0]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "udot z26.s, z10.b, z6.b[0]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "udot z27.s, z11.b, z6.b[0]\n"
+ "b.eq 23f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z24.s, z12.b, z6.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z25.s, z13.b, z6.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "udot z26.s, z14.b, z6.b[1]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "udot z27.s, z15.b, z6.b[1]\n"
+ "b.eq 24f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z24.s, z8.b, z6.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z25.s, z9.b, z6.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "udot z26.s, z10.b, z6.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "udot z27.s, z11.b, z6.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 25f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "26:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "25:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "27:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z20.s, z12.b, z5.b[3]\n"
+ "udot z24.s, z12.b, z6.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z21.s, z13.b, z5.b[3]\n"
+ "udot z25.s, z13.b, z6.b[3]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "udot z22.s, z14.b, z5.b[3]\n"
+ "udot z26.s, z14.b, z6.b[3]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "udot z23.s, z15.b, z5.b[3]\n"
+ "udot z27.s, z15.b, z6.b[3]\n"
+ "b 9f\n"
+ "24:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 28f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "29:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "28:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "30:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z24.s, z8.b, z6.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z25.s, z9.b, z6.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "udot z26.s, z10.b, z6.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "udot z27.s, z11.b, z6.b[2]\n"
+ "b 9f\n"
+ "23:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 31f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "32:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "31:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "33:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z24.s, z12.b, z6.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z25.s, z13.b, z6.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "udot z26.s, z14.b, z6.b[1]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "udot z27.s, z15.b, z6.b[1]\n"
+ "b 9f\n"
+ "22:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 34f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 35f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "35:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "34:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "36:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z24.s, z8.b, z6.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z25.s, z9.b, z6.b[0]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "udot z26.s, z10.b, z6.b[0]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "udot z27.s, z11.b, z6.b[0]\n"
+ "9:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "st1w z24.s, p0, [c_ptr2]\n"
+ "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
+ "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
+ "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
+ );
+ break;
+ default:
+ case 4:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.b, %[temp], %[leftovers]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "whilelt p4.b, %[temp], %[width]\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "incw %[temp], all, mul #1\n"
+ "ptrue p7.b\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "whilelt p1.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p2.s, %[temp], %[width]\n"
+ "incw %[temp], all, mul #1\n"
+ "whilelt p3.s, %[temp], %[width]\n"
+ "cbz %[beta0], 1f\n"
+ "mov z16.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z17.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z18.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z19.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z20.s, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "mov z21.s, #0\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "mov z22.s, #0\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "mov z23.s, #0\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "mov z24.s, #0\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "mov z25.s, #0\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "mov z26.s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "mov z27.s, #0\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "mov z28.s, #0\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 2f\n"
+ "1:\n"
+ "ld1rw z15.s, p7/z, [%[betaptr]]\n"
+ "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
+ "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
+ "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
+ "ld1w z20.s, p0/z, [c_ptr1]\n"
+ "mul z16.s, p7/m, z16.s, z15.s\n"
+ "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
+ "mul z17.s, p7/m, z17.s, z15.s\n"
+ "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
+ "mul z18.s, p7/m, z18.s, z15.s\n"
+ "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
+ "mul z19.s, p7/m, z19.s, z15.s\n"
+ "ld1w z24.s, p0/z, [c_ptr2]\n"
+ "mul z20.s, p7/m, z20.s, z15.s\n"
+ "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
+ "mul z21.s, p7/m, z21.s, z15.s\n"
+ "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
+ "mul z22.s, p7/m, z22.s, z15.s\n"
+ "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
+ "mul z23.s, p7/m, z23.s, z15.s\n"
+ "ld1w z28.s, p0/z, [c_ptr3]\n"
+ "mul z24.s, p7/m, z24.s, z15.s\n"
+ "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
+ "mul z25.s, p7/m, z25.s, z15.s\n"
+ "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
+ "mul z26.s, p7/m, z26.s, z15.s\n"
+ "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
+ "mul z27.s, p7/m, z27.s, z15.s\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mul z28.s, p7/m, z28.s, z15.s\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mul z29.s, p7/m, z29.s, z15.s\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mul z30.s, p7/m, z30.s, z15.s\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mul z31.s, p7/m, z31.s, z15.s\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "add %[a_ptr0], %[a_ptr0], #0x10\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "add a_ptr1, a_ptr1, #0x10\n"
+ "add a_ptr2, a_ptr2, #0x10\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "add a_ptr3, a_ptr3, #0x10\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "2:\n"
+ "cbz %[loops], 3f\n"
+ "4:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "udot z28.s, z8.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr3]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[a_ptr0], %[a_ptr0], #0x20\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "add a_ptr1, a_ptr1, #0x20\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "add a_ptr2, a_ptr2, #0x20\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z29.s, z9.b, z3.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "add a_ptr3, a_ptr3, #0x20\n"
+ "udot z30.s, z10.b, z3.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "udot z31.s, z11.b, z3.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "udot z28.s, z12.b, z3.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "udot z29.s, z13.b, z3.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "udot z30.s, z14.b, z3.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "udot z31.s, z15.b, z3.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "udot z28.s, z8.b, z3.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "udot z29.s, z9.b, z3.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "udot z30.s, z10.b, z3.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "udot z31.s, z11.b, z3.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z24.s, z12.b, z2.b[3]\n"
+ "udot z28.s, z12.b, z3.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z25.s, z13.b, z2.b[3]\n"
+ "udot z29.s, z13.b, z3.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "udot z26.s, z14.b, z2.b[3]\n"
+ "udot z30.s, z14.b, z3.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
+ "udot z27.s, z15.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #-0x10]\n"
+ "udot z31.s, z15.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #-0x10]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z24.s, z8.b, z6.b[0]\n"
+ "udot z28.s, z8.b, z7.b[0]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z25.s, z9.b, z6.b[0]\n"
+ "udot z29.s, z9.b, z7.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "udot z26.s, z10.b, z6.b[0]\n"
+ "udot z30.s, z10.b, z7.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "udot z27.s, z11.b, z6.b[0]\n"
+ "udot z31.s, z11.b, z7.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z24.s, z12.b, z6.b[1]\n"
+ "udot z28.s, z12.b, z7.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z25.s, z13.b, z6.b[1]\n"
+ "udot z29.s, z13.b, z7.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "udot z26.s, z14.b, z6.b[1]\n"
+ "udot z30.s, z14.b, z7.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "udot z27.s, z15.b, z6.b[1]\n"
+ "udot z31.s, z15.b, z7.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z24.s, z8.b, z6.b[2]\n"
+ "udot z28.s, z8.b, z7.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z25.s, z9.b, z6.b[2]\n"
+ "udot z29.s, z9.b, z7.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "udot z26.s, z10.b, z6.b[2]\n"
+ "udot z30.s, z10.b, z7.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "udot z27.s, z11.b, z6.b[2]\n"
+ "udot z31.s, z11.b, z7.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z5.b[3]\n"
+ "udot z24.s, z12.b, z6.b[3]\n"
+ "udot z28.s, z12.b, z7.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z21.s, z13.b, z5.b[3]\n"
+ "udot z25.s, z13.b, z6.b[3]\n"
+ "udot z29.s, z13.b, z7.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "udot z22.s, z14.b, z5.b[3]\n"
+ "udot z26.s, z14.b, z6.b[3]\n"
+ "udot z30.s, z14.b, z7.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "udot z23.s, z15.b, z5.b[3]\n"
+ "udot z27.s, z15.b, z6.b[3]\n"
+ "udot z31.s, z15.b, z7.b[3]\n"
+ "b.ne 4b\n"
+ "3:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "cbz %[regs], 5f\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr1]\n"
+ "udot z28.s, z8.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr2]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr3]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z29.s, z9.b, z3.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "udot z30.s, z10.b, z3.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "udot z31.s, z11.b, z3.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "udot z28.s, z12.b, z3.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "udot z29.s, z13.b, z3.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "udot z30.s, z14.b, z3.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "udot z31.s, z15.b, z3.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "udot z28.s, z8.b, z3.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "udot z29.s, z9.b, z3.b[2]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "udot z30.s, z10.b, z3.b[2]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "udot z31.s, z11.b, z3.b[2]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z24.s, z12.b, z2.b[3]\n"
+ "udot z28.s, z12.b, z3.b[3]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z25.s, z13.b, z2.b[3]\n"
+ "udot z29.s, z13.b, z3.b[3]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "udot z26.s, z14.b, z2.b[3]\n"
+ "udot z30.s, z14.b, z3.b[3]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "udot z27.s, z15.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "udot z31.s, z15.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z24.s, z8.b, z6.b[0]\n"
+ "udot z28.s, z8.b, z7.b[0]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z25.s, z9.b, z6.b[0]\n"
+ "udot z29.s, z9.b, z7.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "udot z26.s, z10.b, z6.b[0]\n"
+ "udot z30.s, z10.b, z7.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "udot z27.s, z11.b, z6.b[0]\n"
+ "udot z31.s, z11.b, z7.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z24.s, z12.b, z6.b[1]\n"
+ "udot z28.s, z12.b, z7.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z25.s, z13.b, z6.b[1]\n"
+ "udot z29.s, z13.b, z7.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "udot z26.s, z14.b, z6.b[1]\n"
+ "udot z30.s, z14.b, z7.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "udot z27.s, z15.b, z6.b[1]\n"
+ "udot z31.s, z15.b, z7.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z24.s, z8.b, z6.b[2]\n"
+ "udot z28.s, z8.b, z7.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z25.s, z9.b, z6.b[2]\n"
+ "udot z29.s, z9.b, z7.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "udot z26.s, z10.b, z6.b[2]\n"
+ "udot z30.s, z10.b, z7.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "udot z27.s, z11.b, z6.b[2]\n"
+ "udot z31.s, z11.b, z7.b[2]\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z20.s, z12.b, z5.b[3]\n"
+ "udot z24.s, z12.b, z6.b[3]\n"
+ "udot z28.s, z12.b, z7.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z21.s, z13.b, z5.b[3]\n"
+ "udot z25.s, z13.b, z6.b[3]\n"
+ "udot z29.s, z13.b, z7.b[3]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "udot z22.s, z14.b, z5.b[3]\n"
+ "udot z26.s, z14.b, z6.b[3]\n"
+ "udot z30.s, z14.b, z7.b[3]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "udot z23.s, z15.b, z5.b[3]\n"
+ "udot z27.s, z15.b, z6.b[3]\n"
+ "udot z31.s, z15.b, z7.b[3]\n"
+ "cbz %[blocks], 6f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "udot z28.s, z8.b, z3.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "udot z29.s, z9.b, z3.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "udot z30.s, z10.b, z3.b[0]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "udot z31.s, z11.b, z3.b[0]\n"
+ "b.eq 7f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "udot z28.s, z12.b, z3.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "udot z29.s, z13.b, z3.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "udot z30.s, z14.b, z3.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "udot z31.s, z15.b, z3.b[1]\n"
+ "b.eq 8f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "udot z28.s, z8.b, z3.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "udot z29.s, z9.b, z3.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "udot z30.s, z10.b, z3.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "udot z31.s, z11.b, z3.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 10f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 11f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "11:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 12f\n"
+ "10:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "12:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z24.s, z12.b, z2.b[3]\n"
+ "udot z28.s, z12.b, z3.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z25.s, z13.b, z2.b[3]\n"
+ "udot z29.s, z13.b, z3.b[3]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "udot z26.s, z14.b, z2.b[3]\n"
+ "udot z30.s, z14.b, z3.b[3]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "udot z27.s, z15.b, z2.b[3]\n"
+ "udot z31.s, z15.b, z3.b[3]\n"
+ "b 9f\n"
+ "8:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 13f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 14f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "14:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 15f\n"
+ "13:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "15:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "udot z28.s, z8.b, z3.b[2]\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "udot z29.s, z9.b, z3.b[2]\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "udot z30.s, z10.b, z3.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "udot z31.s, z11.b, z3.b[2]\n"
+ "b 9f\n"
+ "7:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 16f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 17f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "17:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 18f\n"
+ "16:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "18:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "udot z28.s, z12.b, z3.b[1]\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "udot z29.s, z13.b, z3.b[1]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "udot z30.s, z14.b, z3.b[1]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "udot z31.s, z15.b, z3.b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 19f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 20f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "20:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 21f\n"
+ "19:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "21:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "udot z28.s, z8.b, z3.b[0]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "udot z29.s, z9.b, z3.b[0]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "udot z30.s, z10.b, z3.b[0]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "udot z31.s, z11.b, z3.b[0]\n"
+ "b 9f\n"
+ "5:\n"
+ "udot z16.s, z8.b, z0.b[0]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z20.s, z8.b, z1.b[0]\n"
+ "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
+ "udot z24.s, z8.b, z2.b[0]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr1]\n"
+ "udot z28.s, z8.b, z3.b[0]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr2]\n"
+ "udot z17.s, z9.b, z0.b[0]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr3]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z21.s, z9.b, z1.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z25.s, z9.b, z2.b[0]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "udot z29.s, z9.b, z3.b[0]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "udot z18.s, z10.b, z0.b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "udot z22.s, z10.b, z1.b[0]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "udot z26.s, z10.b, z2.b[0]\n"
+ "udot z30.s, z10.b, z3.b[0]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z11.b, z0.b[0]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "udot z23.s, z11.b, z1.b[0]\n"
+ "udot z27.s, z11.b, z2.b[0]\n"
+ "udot z31.s, z11.b, z3.b[0]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z12.b, z0.b[1]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "udot z20.s, z12.b, z1.b[1]\n"
+ "udot z24.s, z12.b, z2.b[1]\n"
+ "udot z28.s, z12.b, z3.b[1]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "udot z17.s, z13.b, z0.b[1]\n"
+ "udot z21.s, z13.b, z1.b[1]\n"
+ "udot z25.s, z13.b, z2.b[1]\n"
+ "udot z29.s, z13.b, z3.b[1]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "udot z18.s, z14.b, z0.b[1]\n"
+ "udot z22.s, z14.b, z1.b[1]\n"
+ "udot z26.s, z14.b, z2.b[1]\n"
+ "udot z30.s, z14.b, z3.b[1]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "udot z19.s, z15.b, z0.b[1]\n"
+ "udot z23.s, z15.b, z1.b[1]\n"
+ "udot z27.s, z15.b, z2.b[1]\n"
+ "udot z31.s, z15.b, z3.b[1]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "udot z16.s, z8.b, z0.b[2]\n"
+ "udot z20.s, z8.b, z1.b[2]\n"
+ "udot z24.s, z8.b, z2.b[2]\n"
+ "udot z28.s, z8.b, z3.b[2]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "udot z17.s, z9.b, z0.b[2]\n"
+ "udot z21.s, z9.b, z1.b[2]\n"
+ "udot z25.s, z9.b, z2.b[2]\n"
+ "udot z29.s, z9.b, z3.b[2]\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z18.s, z10.b, z0.b[2]\n"
+ "udot z22.s, z10.b, z1.b[2]\n"
+ "udot z26.s, z10.b, z2.b[2]\n"
+ "udot z30.s, z10.b, z3.b[2]\n"
+ "udot z19.s, z11.b, z0.b[2]\n"
+ "udot z23.s, z11.b, z1.b[2]\n"
+ "udot z27.s, z11.b, z2.b[2]\n"
+ "udot z31.s, z11.b, z3.b[2]\n"
+ "udot z16.s, z12.b, z0.b[3]\n"
+ "udot z20.s, z12.b, z1.b[3]\n"
+ "udot z24.s, z12.b, z2.b[3]\n"
+ "udot z28.s, z12.b, z3.b[3]\n"
+ "udot z17.s, z13.b, z0.b[3]\n"
+ "udot z21.s, z13.b, z1.b[3]\n"
+ "udot z25.s, z13.b, z2.b[3]\n"
+ "udot z29.s, z13.b, z3.b[3]\n"
+ "udot z18.s, z14.b, z0.b[3]\n"
+ "udot z22.s, z14.b, z1.b[3]\n"
+ "udot z26.s, z14.b, z2.b[3]\n"
+ "udot z30.s, z14.b, z3.b[3]\n"
+ "udot z19.s, z15.b, z0.b[3]\n"
+ "udot z23.s, z15.b, z1.b[3]\n"
+ "udot z27.s, z15.b, z2.b[3]\n"
+ "udot z31.s, z15.b, z3.b[3]\n"
+ "cbz %[blocks], 22f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z24.s, z8.b, z6.b[0]\n"
+ "udot z28.s, z8.b, z7.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z25.s, z9.b, z6.b[0]\n"
+ "udot z29.s, z9.b, z7.b[0]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "udot z26.s, z10.b, z6.b[0]\n"
+ "udot z30.s, z10.b, z7.b[0]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "udot z27.s, z11.b, z6.b[0]\n"
+ "udot z31.s, z11.b, z7.b[0]\n"
+ "b.eq 23f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "subs %[blocks], %[blocks], #0x1\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "ld1b z12.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z24.s, z12.b, z6.b[1]\n"
+ "udot z28.s, z12.b, z7.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z25.s, z13.b, z6.b[1]\n"
+ "udot z29.s, z13.b, z7.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "udot z26.s, z14.b, z6.b[1]\n"
+ "udot z30.s, z14.b, z7.b[1]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "udot z27.s, z15.b, z6.b[1]\n"
+ "udot z31.s, z15.b, z7.b[1]\n"
+ "b.eq 24f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "add %[b_ptr3], %[b_ptr3], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "ld1b z8.b, p4/z, [%[b_ptr3]]\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z24.s, z8.b, z6.b[2]\n"
+ "udot z28.s, z8.b, z7.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z25.s, z9.b, z6.b[2]\n"
+ "udot z29.s, z9.b, z7.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "udot z26.s, z10.b, z6.b[2]\n"
+ "udot z30.s, z10.b, z7.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "udot z27.s, z11.b, z6.b[2]\n"
+ "udot z31.s, z11.b, z7.b[2]\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 25f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 26f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "26:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 27f\n"
+ "25:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "27:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[3]\n"
+ "udot z20.s, z12.b, z5.b[3]\n"
+ "udot z24.s, z12.b, z6.b[3]\n"
+ "udot z28.s, z12.b, z7.b[3]\n"
+ "udot z17.s, z13.b, z4.b[3]\n"
+ "udot z21.s, z13.b, z5.b[3]\n"
+ "udot z25.s, z13.b, z6.b[3]\n"
+ "udot z29.s, z13.b, z7.b[3]\n"
+ "udot z18.s, z14.b, z4.b[3]\n"
+ "udot z22.s, z14.b, z5.b[3]\n"
+ "udot z26.s, z14.b, z6.b[3]\n"
+ "udot z30.s, z14.b, z7.b[3]\n"
+ "udot z19.s, z15.b, z4.b[3]\n"
+ "udot z23.s, z15.b, z5.b[3]\n"
+ "udot z27.s, z15.b, z6.b[3]\n"
+ "udot z31.s, z15.b, z7.b[3]\n"
+ "b 9f\n"
+ "24:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 28f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 29f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "29:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 30f\n"
+ "28:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "30:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[2]\n"
+ "udot z20.s, z8.b, z5.b[2]\n"
+ "udot z24.s, z8.b, z6.b[2]\n"
+ "udot z28.s, z8.b, z7.b[2]\n"
+ "udot z17.s, z9.b, z4.b[2]\n"
+ "udot z21.s, z9.b, z5.b[2]\n"
+ "udot z25.s, z9.b, z6.b[2]\n"
+ "udot z29.s, z9.b, z7.b[2]\n"
+ "udot z18.s, z10.b, z4.b[2]\n"
+ "udot z22.s, z10.b, z5.b[2]\n"
+ "udot z26.s, z10.b, z6.b[2]\n"
+ "udot z30.s, z10.b, z7.b[2]\n"
+ "udot z19.s, z11.b, z4.b[2]\n"
+ "udot z23.s, z11.b, z5.b[2]\n"
+ "udot z27.s, z11.b, z6.b[2]\n"
+ "udot z31.s, z11.b, z7.b[2]\n"
+ "b 9f\n"
+ "23:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 31f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 32f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z13.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "32:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z14.b, p4/z, [%[b_ptr1]]\n"
+ "b 33f\n"
+ "31:\n"
+ "mov z13.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z14.b, #0\n"
+ "ld1b z12.b, p4/z, [%[b_ptr0]]\n"
+ "33:\n"
+ "zip2 z15.b, z12.b, z13.b\n"
+ "zip1 z13.b, z12.b, z13.b\n"
+ "mov z12.b, #0\n"
+ "zip2 z8.b, z14.b, z12.b\n"
+ "zip1 z14.b, z14.b, z12.b\n"
+ "zip1 z12.b, z13.b, z14.b\n"
+ "zip2 z13.b, z13.b, z14.b\n"
+ "zip1 z14.b, z15.b, z8.b\n"
+ "zip2 z15.b, z15.b, z8.b\n"
+ "udot z16.s, z12.b, z4.b[1]\n"
+ "udot z20.s, z12.b, z5.b[1]\n"
+ "udot z24.s, z12.b, z6.b[1]\n"
+ "udot z28.s, z12.b, z7.b[1]\n"
+ "udot z17.s, z13.b, z4.b[1]\n"
+ "udot z21.s, z13.b, z5.b[1]\n"
+ "udot z25.s, z13.b, z6.b[1]\n"
+ "udot z29.s, z13.b, z7.b[1]\n"
+ "udot z18.s, z14.b, z4.b[1]\n"
+ "udot z22.s, z14.b, z5.b[1]\n"
+ "udot z26.s, z14.b, z6.b[1]\n"
+ "udot z30.s, z14.b, z7.b[1]\n"
+ "udot z19.s, z15.b, z4.b[1]\n"
+ "udot z23.s, z15.b, z5.b[1]\n"
+ "udot z27.s, z15.b, z6.b[1]\n"
+ "udot z31.s, z15.b, z7.b[1]\n"
+ "b 9f\n"
+ "22:\n"
+ "cbz %[odds], 9f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 34f\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "b.eq 35f\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr2], %[b_ptr2], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z9.b, p4/z, [%[b_ptr2]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "35:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add %[b_ptr1], %[b_ptr1], %[ldb]\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "ld1b z10.b, p4/z, [%[b_ptr1]]\n"
+ "b 36f\n"
+ "34:\n"
+ "mov z9.b, #0\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "mov z10.b, #0\n"
+ "ld1b z8.b, p4/z, [%[b_ptr0]]\n"
+ "36:\n"
+ "zip2 z11.b, z8.b, z9.b\n"
+ "zip1 z9.b, z8.b, z9.b\n"
+ "mov z8.b, #0\n"
+ "zip2 z12.b, z10.b, z8.b\n"
+ "zip1 z10.b, z10.b, z8.b\n"
+ "zip1 z8.b, z9.b, z10.b\n"
+ "zip2 z9.b, z9.b, z10.b\n"
+ "zip1 z10.b, z11.b, z12.b\n"
+ "zip2 z11.b, z11.b, z12.b\n"
+ "udot z16.s, z8.b, z4.b[0]\n"
+ "udot z20.s, z8.b, z5.b[0]\n"
+ "udot z24.s, z8.b, z6.b[0]\n"
+ "udot z28.s, z8.b, z7.b[0]\n"
+ "udot z17.s, z9.b, z4.b[0]\n"
+ "udot z21.s, z9.b, z5.b[0]\n"
+ "udot z25.s, z9.b, z6.b[0]\n"
+ "udot z29.s, z9.b, z7.b[0]\n"
+ "udot z18.s, z10.b, z4.b[0]\n"
+ "udot z22.s, z10.b, z5.b[0]\n"
+ "udot z26.s, z10.b, z6.b[0]\n"
+ "udot z30.s, z10.b, z7.b[0]\n"
+ "udot z19.s, z11.b, z4.b[0]\n"
+ "udot z23.s, z11.b, z5.b[0]\n"
+ "udot z27.s, z11.b, z6.b[0]\n"
+ "udot z31.s, z11.b, z7.b[0]\n"
+ "9:\n"
+ "st1w z16.s, p0, [%[c_ptr0]]\n"
+ "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
+ "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
+ "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #4\n"
+ "st1w z20.s, p0, [c_ptr1]\n"
+ "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
+ "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
+ "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
+ "st1w z24.s, p0, [c_ptr2]\n"
+ "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
+ "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
+ "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
+ "st1w z28.s, p0, [c_ptr3]\n"
+ "st1w z29.s, p1, [c_ptr3, #1, MUL VL]\n"
+ "st1w z30.s, p2, [c_ptr3, #2, MUL VL]\n"
+ "st1w z31.s, p3, [c_ptr3, #3, MUL VL]\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
+ );
+ break;
+ }
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp
new file mode 100644
index 0000000..80b216c
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+
+
+namespace arm_gemm
+{
+
+// Actual kernel implementations
+void sve_smallK_fp32_mla_1VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+
+class smallK_fp32_mla_1VLx4
+{
+public:
+ typedef float operand_type;
+ typedef float result_type;
+
+ typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int);
+
+ /* Kernel blocking parameters */
+ static int out_height()
+ {
+ return 4;
+ }
+
+ static int out_width()
+ {
+ return get_vector_length<float>() * 1;
+ }
+
+ static int k_unroll()
+ {
+ return 1;
+ }
+
+
+
+ // Default to the generic kernel
+ kern_type kernel=sve_smallK_fp32_mla_1VLx4;
+
+ smallK_fp32_mla_1VLx4(const CPUInfo *ci)
+ {
+
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp
new file mode 100644
index 0000000..e2cc1d1
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp
@@ -0,0 +1,4264 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include <algorithm>
+
+
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+namespace arm_gemm {
+
+void sve_smallK_fp32_mla_1VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) {
+ const long beta0 = (beta == 0.0f);
+
+ const long loops_count = M / 4;
+ const long oddrow_count = M % 4;
+ const long ldab = lda * sizeof(float);
+ const long ldcb = ldc * sizeof(float);
+ const long odd_depth = K % 4;
+ const float *betaptr = β
+ long ldbb = ldb * sizeof(float);
+
+ for (int x0=0; x0<N; x0+=(get_vector_length<float>() * 1)) {
+ const long width = std::min((unsigned long)N-x0, (get_vector_length<float>() * 1));
+ long loops = loops_count;
+ long oddrows = oddrow_count;
+ long temp = 0;
+ const float *b_ptr0 = B + x0;
+
+ const float *a_ptr0 = A;
+
+ float *c_ptr0 = C + x0;
+
+ switch(K) {
+ case 1:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 2:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 3:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 4:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 5:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 6:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 7:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 8:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 9:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 10:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 11:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 12:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 13:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 14:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 15:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 16:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 17:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 18:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 19:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 20:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 21:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z24.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
+ "fmla z28.s, z24.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z24.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z24.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z24.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z28.s, z24.s, z1.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 22:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z24.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z25.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
+ "fmla z28.s, z24.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z24.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z24.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z24.s, z3.s[0]\n"
+ "fmla z28.s, z25.s, z0.s[1]\n"
+ "fmla z29.s, z25.s, z1.s[1]\n"
+ "fmla z30.s, z25.s, z2.s[1]\n"
+ "fmla z31.s, z25.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z28.s, z24.s, z1.s[0]\n"
+ "fmla z28.s, z25.s, z1.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 23:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z24.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z25.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z26.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
+ "fmla z28.s, z24.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z24.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z24.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z24.s, z3.s[0]\n"
+ "fmla z28.s, z25.s, z0.s[1]\n"
+ "fmla z29.s, z25.s, z1.s[1]\n"
+ "fmla z30.s, z25.s, z2.s[1]\n"
+ "fmla z31.s, z25.s, z3.s[1]\n"
+ "fmla z28.s, z26.s, z0.s[2]\n"
+ "fmla z29.s, z26.s, z1.s[2]\n"
+ "fmla z30.s, z26.s, z2.s[2]\n"
+ "fmla z31.s, z26.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z28.s, z24.s, z1.s[0]\n"
+ "fmla z28.s, z25.s, z1.s[1]\n"
+ "fmla z28.s, z26.s, z1.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ default:
+ case 24:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z5.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z6.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z7.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z8.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z9.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z10.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z11.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z12.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z13.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z14.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z15.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z16.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z17.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z18.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z19.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z20.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z21.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z22.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z23.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z24.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z25.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z26.s, p0/z, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], %[ldb]\n"
+ "ld1w z27.s, p0/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n"
+ "fmla z28.s, z24.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z24.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z24.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z24.s, z3.s[0]\n"
+ "fmla z28.s, z25.s, z0.s[1]\n"
+ "fmla z29.s, z25.s, z1.s[1]\n"
+ "fmla z30.s, z25.s, z2.s[1]\n"
+ "fmla z31.s, z25.s, z3.s[1]\n"
+ "fmla z28.s, z26.s, z0.s[2]\n"
+ "fmla z29.s, z26.s, z1.s[2]\n"
+ "fmla z30.s, z26.s, z2.s[2]\n"
+ "fmla z31.s, z26.s, z3.s[2]\n"
+ "fmla z28.s, z27.s, z0.s[3]\n"
+ "fmla z29.s, z27.s, z1.s[3]\n"
+ "fmla z30.s, z27.s, z2.s[3]\n"
+ "fmla z31.s, z27.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z28.s, z24.s, z1.s[0]\n"
+ "fmla z28.s, z25.s, z1.s[1]\n"
+ "fmla z28.s, z26.s, z1.s[2]\n"
+ "fmla z28.s, z27.s, z1.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp
new file mode 100644
index 0000000..aa2c522
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+
+
+namespace arm_gemm
+{
+
+// Actual kernel implementations
+void sve_smallK_hybrid_fp32_mla_1VLx4(const float *, int, const float *, float *, int, float, int, int, int);
+
+class smallK_hybrid_fp32_mla_1VLx4
+{
+public:
+ typedef float operand_type;
+ typedef float result_type;
+
+ typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int);
+
+ /* Kernel blocking parameters */
+ static int out_height()
+ {
+ return 4;
+ }
+
+ static int out_width()
+ {
+ return get_vector_length<float>() * 1;
+ }
+
+ static int k_unroll()
+ {
+ return 1;
+ }
+
+ StdTransformsSVE<operand_type, result_type, 4, 1, 1> transforms = {};
+
+ // Default to the generic kernel
+ kern_type kernel=sve_smallK_hybrid_fp32_mla_1VLx4;
+
+ smallK_hybrid_fp32_mla_1VLx4(const CPUInfo *ci)
+ {
+
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp
new file mode 100644
index 0000000..3e7e713
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp
@@ -0,0 +1,4004 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include <algorithm>
+
+
+#include "../../asmlib.hpp"
+#include "../../utils.hpp"
+
+namespace arm_gemm {
+
+void sve_smallK_hybrid_fp32_mla_1VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) {
+ const long beta0 = (beta == 0.0f);
+
+ const long loops_count = M / 4;
+ const long oddrow_count = M % 4;
+ const long ldab = lda * sizeof(float);
+ const long ldcb = ldc * sizeof(float);
+ const int K_stride = K;
+ const long odd_depth = K % 4;
+ const float *betaptr = β
+
+ for (int x0=0; x0<N; x0+=(get_vector_length<float>() * 1)) {
+ const long width = std::min((unsigned long)N-x0, (get_vector_length<float>() * 1));
+ long loops = loops_count;
+ long oddrows = oddrow_count;
+ long temp = 0;
+ const float *b_ptr0 = B + (K_stride * x0);
+
+ const float *a_ptr0 = A;
+
+ float *c_ptr0 = C + x0;
+
+ switch(K) {
+ case 1:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 2:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 3:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 4:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 5:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 6:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 7:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 8:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 9:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 10:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 11:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 12:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 13:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 14:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 15:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 16:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 17:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 18:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 19:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 20:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 21:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
+ "fmla z28.s, z24.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z24.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z24.s, z2.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "fmla z31.s, z24.s, z3.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z28.s, z24.s, z1.s[0]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 22:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
+ "fmla z28.s, z24.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z24.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z24.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z24.s, z3.s[0]\n"
+ "fmla z28.s, z25.s, z0.s[1]\n"
+ "fmla z29.s, z25.s, z1.s[1]\n"
+ "fmla z30.s, z25.s, z2.s[1]\n"
+ "fmla z31.s, z25.s, z3.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z28.s, z24.s, z1.s[0]\n"
+ "fmla z28.s, z25.s, z1.s[1]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ case 23:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n"
+ "fmla z28.s, z24.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z24.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z24.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z24.s, z3.s[0]\n"
+ "fmla z28.s, z25.s, z0.s[1]\n"
+ "fmla z29.s, z25.s, z1.s[1]\n"
+ "fmla z30.s, z25.s, z2.s[1]\n"
+ "fmla z31.s, z25.s, z3.s[1]\n"
+ "fmla z28.s, z26.s, z0.s[2]\n"
+ "fmla z29.s, z26.s, z1.s[2]\n"
+ "fmla z30.s, z26.s, z2.s[2]\n"
+ "fmla z31.s, z26.s, z3.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z28.s, z24.s, z1.s[0]\n"
+ "fmla z28.s, z25.s, z1.s[1]\n"
+ "fmla z28.s, z26.s, z1.s[2]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ default:
+ case 24:
+ __asm __volatile (
+ "a_ptr1 .req X0\n"
+ "a_ptr2 .req X1\n"
+ "a_ptr3 .req X2\n"
+ "c_ptr1 .req X3\n"
+ "c_ptr2 .req X4\n"
+ "c_ptr3 .req X5\n"
+ "add a_ptr1, %[a_ptr0], %[lda]\n"
+ "add c_ptr1, %[c_ptr0], %[ldc]\n"
+ "whilelt p6.s, %[temp], %[odd_depth]\n"
+ "whilelt p0.s, %[temp], %[width]\n"
+ "ptrue p7.s\n"
+ "add a_ptr2, a_ptr1, %[lda]\n"
+ "add c_ptr2, c_ptr1, %[ldc]\n"
+ "ld1w z4.s, p7/z, [%[b_ptr0]]\n"
+ "add a_ptr3, a_ptr2, %[lda]\n"
+ "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "add c_ptr3, c_ptr2, %[ldc]\n"
+ "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #16\n"
+ "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
+ "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
+ "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
+ "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
+ "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
+ "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
+ "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
+ "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
+ "ld1w z20.s, p7/z, [%[b_ptr0]]\n"
+ "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1w z27.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ "cbz %[beta0], 3f\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "b 4f\n"
+ "3:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "ld1w z29.s, p0/z, [c_ptr1]\n"
+ "ld1w z30.s, p0/z, [c_ptr2]\n"
+ "ld1w z31.s, p0/z, [c_ptr3]\n"
+ "4:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z29.s, z4.s, z1.s[0]\n"
+ "fmla z30.s, z4.s, z2.s[0]\n"
+ "fmla z31.s, z4.s, z3.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z29.s, z5.s, z1.s[1]\n"
+ "fmla z30.s, z5.s, z2.s[1]\n"
+ "fmla z31.s, z5.s, z3.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z29.s, z6.s, z1.s[2]\n"
+ "fmla z30.s, z6.s, z2.s[2]\n"
+ "fmla z31.s, z6.s, z3.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "fmla z29.s, z7.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n"
+ "fmla z30.s, z7.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n"
+ "fmla z31.s, z7.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n"
+ "fmla z28.s, z8.s, z0.s[0]\n"
+ "fmla z29.s, z8.s, z1.s[0]\n"
+ "fmla z30.s, z8.s, z2.s[0]\n"
+ "fmla z31.s, z8.s, z3.s[0]\n"
+ "fmla z28.s, z9.s, z0.s[1]\n"
+ "fmla z29.s, z9.s, z1.s[1]\n"
+ "fmla z30.s, z9.s, z2.s[1]\n"
+ "fmla z31.s, z9.s, z3.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[2]\n"
+ "fmla z29.s, z10.s, z1.s[2]\n"
+ "fmla z30.s, z10.s, z2.s[2]\n"
+ "fmla z31.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z11.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "fmla z29.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n"
+ "fmla z30.s, z11.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n"
+ "fmla z31.s, z11.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n"
+ "fmla z28.s, z12.s, z0.s[0]\n"
+ "fmla z29.s, z12.s, z1.s[0]\n"
+ "fmla z30.s, z12.s, z2.s[0]\n"
+ "fmla z31.s, z12.s, z3.s[0]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z1.s[1]\n"
+ "fmla z30.s, z13.s, z2.s[1]\n"
+ "fmla z31.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z14.s, z0.s[2]\n"
+ "fmla z29.s, z14.s, z1.s[2]\n"
+ "fmla z30.s, z14.s, z2.s[2]\n"
+ "fmla z31.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z29.s, z15.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n"
+ "fmla z30.s, z15.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n"
+ "fmla z31.s, z15.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n"
+ "fmla z28.s, z16.s, z0.s[0]\n"
+ "fmla z29.s, z16.s, z1.s[0]\n"
+ "fmla z30.s, z16.s, z2.s[0]\n"
+ "fmla z31.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z17.s, z1.s[1]\n"
+ "fmla z30.s, z17.s, z2.s[1]\n"
+ "fmla z31.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z1.s[2]\n"
+ "fmla z30.s, z18.s, z2.s[2]\n"
+ "fmla z31.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z29.s, z19.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n"
+ "fmla z30.s, z19.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n"
+ "fmla z31.s, z19.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z29.s, z20.s, z1.s[0]\n"
+ "fmla z30.s, z20.s, z2.s[0]\n"
+ "fmla z31.s, z20.s, z3.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z29.s, z21.s, z1.s[1]\n"
+ "fmla z30.s, z21.s, z2.s[1]\n"
+ "fmla z31.s, z21.s, z3.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z29.s, z22.s, z1.s[2]\n"
+ "fmla z30.s, z22.s, z2.s[2]\n"
+ "fmla z31.s, z22.s, z3.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n"
+ "fmla z29.s, z23.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n"
+ "fmla z30.s, z23.s, z2.s[3]\n"
+ "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n"
+ "fmla z31.s, z23.s, z3.s[3]\n"
+ "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n"
+ "fmla z28.s, z24.s, z0.s[0]\n"
+ "add a_ptr1, a_ptr1, %[lda], LSL #2\n"
+ "fmla z29.s, z24.s, z1.s[0]\n"
+ "add a_ptr2, a_ptr2, %[lda], LSL #2\n"
+ "fmla z30.s, z24.s, z2.s[0]\n"
+ "add a_ptr3, a_ptr3, %[lda], LSL #2\n"
+ "fmla z31.s, z24.s, z3.s[0]\n"
+ "fmla z28.s, z25.s, z0.s[1]\n"
+ "fmla z29.s, z25.s, z1.s[1]\n"
+ "fmla z30.s, z25.s, z2.s[1]\n"
+ "fmla z31.s, z25.s, z3.s[1]\n"
+ "fmla z28.s, z26.s, z0.s[2]\n"
+ "fmla z29.s, z26.s, z1.s[2]\n"
+ "fmla z30.s, z26.s, z2.s[2]\n"
+ "fmla z31.s, z26.s, z3.s[2]\n"
+ "fmla z28.s, z27.s, z0.s[3]\n"
+ "fmla z29.s, z27.s, z1.s[3]\n"
+ "fmla z30.s, z27.s, z2.s[3]\n"
+ "fmla z31.s, z27.s, z3.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n"
+ "st1w z29.s, p0, [c_ptr1]\n"
+ "add c_ptr1, c_ptr1, %[ldc], LSL #2\n"
+ "st1w z30.s, p0, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, %[ldc], LSL #2\n"
+ "st1w z31.s, p0, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, %[ldc], LSL #2\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[oddrows], 5f\n"
+ "6:\n"
+ "cbz %[beta0], 7f\n"
+ "mov z28.s, #0\n"
+ "b 8f\n"
+ "7:\n"
+ "ld1w z28.s, p0/z, [%[c_ptr0]]\n"
+ "8:\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
+ "subs %[oddrows], %[oddrows], #0x1\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n"
+ "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n"
+ "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n"
+ "fmla z28.s, z4.s, z0.s[0]\n"
+ "fmla z28.s, z5.s, z0.s[1]\n"
+ "fmla z28.s, z6.s, z0.s[2]\n"
+ "fmla z28.s, z7.s, z0.s[3]\n"
+ "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n"
+ "fmla z28.s, z8.s, z1.s[0]\n"
+ "fmla z28.s, z9.s, z1.s[1]\n"
+ "fmla z28.s, z10.s, z1.s[2]\n"
+ "fmla z28.s, z11.s, z1.s[3]\n"
+ "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n"
+ "add %[a_ptr0], %[a_ptr0], %[lda]\n"
+ "fmla z28.s, z12.s, z2.s[0]\n"
+ "fmla z28.s, z13.s, z2.s[1]\n"
+ "fmla z28.s, z14.s, z2.s[2]\n"
+ "fmla z28.s, z15.s, z2.s[3]\n"
+ "fmla z28.s, z16.s, z3.s[0]\n"
+ "fmla z28.s, z17.s, z3.s[1]\n"
+ "fmla z28.s, z18.s, z3.s[2]\n"
+ "fmla z28.s, z19.s, z3.s[3]\n"
+ "fmla z28.s, z20.s, z0.s[0]\n"
+ "fmla z28.s, z21.s, z0.s[1]\n"
+ "fmla z28.s, z22.s, z0.s[2]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z28.s, z24.s, z1.s[0]\n"
+ "fmla z28.s, z25.s, z1.s[1]\n"
+ "fmla z28.s, z26.s, z1.s[2]\n"
+ "fmla z28.s, z27.s, z1.s[3]\n"
+ "st1w z28.s, p0, [%[c_ptr0]]\n"
+ "add %[c_ptr0], %[c_ptr0], %[ldc]\n"
+ "b.ne 6b\n"
+ "5:\n"
+ ".unreq a_ptr1\n"
+ ".unreq a_ptr2\n"
+ ".unreq a_ptr3\n"
+ ".unreq c_ptr1\n"
+ ".unreq c_ptr2\n"
+ ".unreq c_ptr3\n"
+ : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows)
+ : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ break;
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp
new file mode 100644
index 0000000..fcdca59
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp
@@ -0,0 +1,1660 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __aarch64__
+
+template<>
+inline void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta)
+{
+ const float *inptr = in;
+
+ for (int y=y0; y<ymax; y+=8) {
+ float *outptr0 = out + (y * ldout) + x0;
+ float *outptr1 = outptr0 + ldout;
+ float *outptr2 = outptr1 + ldout;
+ float *outptr3 = outptr2 + ldout;
+ float *outptr4 = outptr3 + ldout;
+ float *outptr5 = outptr4 + ldout;
+ float *outptr6 = outptr5 + ldout;
+ float *outptr7 = outptr6 + ldout;
+
+ const int height = ymax - y;
+
+ for (int i=x0; i<xmax; i+=12) {
+ if (beta==0.0f)
+ {
+ switch(height) {
+ case 1:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]);
+ outptr0++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q4, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q5, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr0], #0x10]\n"
+ "ldr q6, [%[inptr], #0x20]\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]);
+ outptr1++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q4, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q6, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x10]\n"
+ "ldr q7, [%[inptr], #0x40]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x10]\n"
+ "ldr q4, [%[inptr], #0x20]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x20]\n"
+ "ldr q5, [%[inptr], #0x50]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]);
+ outptr2++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q4, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q7, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q4, [%[inptr], #0x40]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr1], #0x10]\n"
+ "ldr q5, [%[inptr], #0x70]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr2], #0x10]\n"
+ "ldr q6, [%[inptr], #0x20]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x20]\n"
+ "ldr q7, [%[inptr], #0x50]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x20]\n"
+ "ldr q4, [%[inptr], #0x80]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 4:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]);
+ outptr3++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q4, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q4, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x10]\n"
+ "ldr q5, [%[inptr], #0x40]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x10]\n"
+ "ldr q6, [%[inptr], #0x70]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x10]\n"
+ "ldr q7, [%[inptr], #0xa0]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x10]\n"
+ "ldr q4, [%[inptr], #0x20]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x20]\n"
+ "ldr q5, [%[inptr], #0x50]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x20]\n"
+ "ldr q6, [%[inptr], #0x80]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q7, [%[inptr], #0xb0]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 5:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]);
+ outptr3++;
+ *outptr4 = (alpha * inptr[xi + 48]);
+ outptr4++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q4, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q4, [%[inptr], #0xc0]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4]]\n"
+ "ldr q5, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr0], #0x10]\n"
+ "ldr q6, [%[inptr], #0x40]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr1], #0x10]\n"
+ "ldr q7, [%[inptr], #0x70]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr2], #0x10]\n"
+ "ldr q4, [%[inptr], #0xa0]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr3], #0x10]\n"
+ "ldr q5, [%[inptr], #0xd0]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr4], #0x10]\n"
+ "ldr q6, [%[inptr], #0x20]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x20]\n"
+ "ldr q7, [%[inptr], #0x50]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x20]\n"
+ "ldr q4, [%[inptr], #0x80]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr2], #0x20]\n"
+ "ldr q5, [%[inptr], #0xb0]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr3], #0x20]\n"
+ "ldr q6, [%[inptr], #0xe0]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 6:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]);
+ outptr3++;
+ *outptr4 = (alpha * inptr[xi + 48]);
+ outptr4++;
+ *outptr5 = (alpha * inptr[xi + 60]);
+ outptr5++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q4, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q4, [%[inptr], #0xc0]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4]]\n"
+ "ldr q5, [%[inptr], #0xf0]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5]]\n"
+ "ldr q6, [%[inptr], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x10]\n"
+ "ldr q7, [%[inptr], #0x40]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x10]\n"
+ "ldr q4, [%[inptr], #0x70]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr2], #0x10]\n"
+ "ldr q5, [%[inptr], #0xa0]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr3], #0x10]\n"
+ "ldr q6, [%[inptr], #0xd0]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr4], #0x10]\n"
+ "ldr q7, [%[inptr], #0x100]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr5], #0x10]\n"
+ "ldr q4, [%[inptr], #0x20]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x20]\n"
+ "ldr q5, [%[inptr], #0x50]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x20]\n"
+ "ldr q6, [%[inptr], #0x80]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q7, [%[inptr], #0xb0]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x20]\n"
+ "ldr q4, [%[inptr], #0xe0]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4], #0x20]\n"
+ "ldr q5, [%[inptr], #0x110]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 7:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]);
+ outptr3++;
+ *outptr4 = (alpha * inptr[xi + 48]);
+ outptr4++;
+ *outptr5 = (alpha * inptr[xi + 60]);
+ outptr5++;
+ *outptr6 = (alpha * inptr[xi + 72]);
+ outptr6++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q4, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q4, [%[inptr], #0xc0]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4]]\n"
+ "ldr q5, [%[inptr], #0xf0]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5]]\n"
+ "ldr q6, [%[inptr], #0x120]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr6]]\n"
+ "ldr q7, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q4, [%[inptr], #0x40]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr1], #0x10]\n"
+ "ldr q5, [%[inptr], #0x70]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr2], #0x10]\n"
+ "ldr q6, [%[inptr], #0xa0]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr3], #0x10]\n"
+ "ldr q7, [%[inptr], #0xd0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr4], #0x10]\n"
+ "ldr q4, [%[inptr], #0x100]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr5], #0x10]\n"
+ "ldr q5, [%[inptr], #0x130]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr6], #0x10]\n"
+ "ldr q6, [%[inptr], #0x20]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x20]\n"
+ "ldr q7, [%[inptr], #0x50]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x20]\n"
+ "ldr q4, [%[inptr], #0x80]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr2], #0x20]\n"
+ "ldr q5, [%[inptr], #0xb0]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr3], #0x20]\n"
+ "ldr q6, [%[inptr], #0xe0]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr4], #0x20]\n"
+ "ldr q7, [%[inptr], #0x110]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "ldr q4, [%[inptr], #0x140]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]);
+ outptr3++;
+ *outptr4 = (alpha * inptr[xi + 48]);
+ outptr4++;
+ *outptr5 = (alpha * inptr[xi + 60]);
+ outptr5++;
+ *outptr6 = (alpha * inptr[xi + 72]);
+ outptr6++;
+ *outptr7 = (alpha * inptr[xi + 84]);
+ outptr7++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q4, [%[inptr]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q4, [%[inptr], #0xc0]\n"
+ "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4]]\n"
+ "ldr q5, [%[inptr], #0xf0]\n"
+ "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5]]\n"
+ "ldr q6, [%[inptr], #0x120]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr6]]\n"
+ "ldr q7, [%[inptr], #0x150]\n"
+ "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr7]]\n"
+ "ldr q4, [%[inptr], #0x10]\n"
+ "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x10]\n"
+ "ldr q5, [%[inptr], #0x40]\n"
+ "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x10]\n"
+ "ldr q6, [%[inptr], #0x70]\n"
+ "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x10]\n"
+ "ldr q7, [%[inptr], #0xa0]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x10]\n"
+ "ldr q4, [%[inptr], #0xd0]\n"
+ "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4], #0x10]\n"
+ "ldr q5, [%[inptr], #0x100]\n"
+ "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5], #0x10]\n"
+ "ldr q6, [%[inptr], #0x130]\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr6], #0x10]\n"
+ "ldr q7, [%[inptr], #0x160]\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr7], #0x10]\n"
+ "ldr q4, [%[inptr], #0x20]\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x20]\n"
+ "ldr q5, [%[inptr], #0x50]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x20]\n"
+ "ldr q6, [%[inptr], #0x80]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q7, [%[inptr], #0xb0]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x20]\n"
+ "ldr q4, [%[inptr], #0xe0]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmul v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4], #0x20]\n"
+ "ldr q5, [%[inptr], #0x110]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmul v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5], #0x20]\n"
+ "ldr q6, [%[inptr], #0x140]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "fmul v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr6], #0x20]\n"
+ "ldr q7, [%[inptr], #0x170]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "fmul v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr7], #0x20]\n"
+ "add %[outptr7], %[outptr7], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ else
+ {
+ switch(height) {
+ case 1:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ outptr0++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q8, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr]]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q9, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x10]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr0], #0x10]\n"
+ "ldr q10, [%[outptr0], #0x20]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x20]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 2:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ outptr1++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q8, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr]]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q9, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q10, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x10]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x10]\n"
+ "ldr q11, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x40]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x10]\n"
+ "ldr q8, [%[outptr0], #0x20]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x20]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x20]\n"
+ "ldr q9, [%[outptr1], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x50]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 3:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ outptr2++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q8, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr]]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q9, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q10, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q11, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x10]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q8, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x40]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr1], #0x10]\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x70]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr2], #0x10]\n"
+ "ldr q10, [%[outptr0], #0x20]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x20]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x20]\n"
+ "ldr q11, [%[outptr1], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x50]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x20]\n"
+ "ldr q8, [%[outptr2], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x80]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr2], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 4:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ outptr3++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q8, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr]]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q9, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q10, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q11, [%[outptr3]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q8, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x10]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x10]\n"
+ "ldr q9, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x40]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x10]\n"
+ "ldr q10, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x70]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x10]\n"
+ "ldr q11, [%[outptr3], #0x10]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0xa0]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x10]\n"
+ "ldr q8, [%[outptr0], #0x20]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x20]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x20]\n"
+ "ldr q9, [%[outptr1], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x50]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x20]\n"
+ "ldr q10, [%[outptr2], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x80]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q11, [%[outptr3], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0xb0]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 5:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ outptr3++;
+ *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
+ outptr4++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q8, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr]]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q9, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q10, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q11, [%[outptr3]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q8, [%[outptr4]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0xc0]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4]]\n"
+ "ldr q9, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x10]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr0], #0x10]\n"
+ "ldr q10, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x40]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr1], #0x10]\n"
+ "ldr q11, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x70]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr2], #0x10]\n"
+ "ldr q8, [%[outptr3], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0xa0]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr3], #0x10]\n"
+ "ldr q9, [%[outptr4], #0x10]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0xd0]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr4], #0x10]\n"
+ "ldr q10, [%[outptr0], #0x20]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x20]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x20]\n"
+ "ldr q11, [%[outptr1], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x50]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x20]\n"
+ "ldr q8, [%[outptr2], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x80]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr2], #0x20]\n"
+ "ldr q9, [%[outptr3], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0xb0]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr3], #0x20]\n"
+ "ldr q10, [%[outptr4], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0xe0]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr4], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 6:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ outptr3++;
+ *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
+ outptr4++;
+ *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta);
+ outptr5++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q8, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr]]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q9, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q10, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q11, [%[outptr3]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q8, [%[outptr4]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0xc0]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4]]\n"
+ "ldr q9, [%[outptr5]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0xf0]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5]]\n"
+ "ldr q10, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x10]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x10]\n"
+ "ldr q11, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x40]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x10]\n"
+ "ldr q8, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x70]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr2], #0x10]\n"
+ "ldr q9, [%[outptr3], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0xa0]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr3], #0x10]\n"
+ "ldr q10, [%[outptr4], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0xd0]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr4], #0x10]\n"
+ "ldr q11, [%[outptr5], #0x10]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x100]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr5], #0x10]\n"
+ "ldr q8, [%[outptr0], #0x20]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x20]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x20]\n"
+ "ldr q9, [%[outptr1], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x50]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x20]\n"
+ "ldr q10, [%[outptr2], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x80]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q11, [%[outptr3], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0xb0]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x20]\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0xe0]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4], #0x20]\n"
+ "ldr q9, [%[outptr5], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x110]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ case 7:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ outptr3++;
+ *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
+ outptr4++;
+ *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta);
+ outptr5++;
+ *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta);
+ outptr6++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q8, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr]]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q9, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q10, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q11, [%[outptr3]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q8, [%[outptr4]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0xc0]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4]]\n"
+ "ldr q9, [%[outptr5]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0xf0]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5]]\n"
+ "ldr q10, [%[outptr6]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x120]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr6]]\n"
+ "ldr q11, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x10]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr0], #0x10]\n"
+ "ldr q8, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x40]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr1], #0x10]\n"
+ "ldr q9, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x70]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr2], #0x10]\n"
+ "ldr q10, [%[outptr3], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0xa0]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr3], #0x10]\n"
+ "ldr q11, [%[outptr4], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0xd0]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr4], #0x10]\n"
+ "ldr q8, [%[outptr5], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x100]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr5], #0x10]\n"
+ "ldr q9, [%[outptr6], #0x10]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x130]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr6], #0x10]\n"
+ "ldr q10, [%[outptr0], #0x20]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x20]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr0], #0x20]\n"
+ "ldr q11, [%[outptr1], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x50]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr1], #0x20]\n"
+ "ldr q8, [%[outptr2], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x80]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr2], #0x20]\n"
+ "ldr q9, [%[outptr3], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0xb0]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr3], #0x20]\n"
+ "ldr q10, [%[outptr4], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0xe0]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr4], #0x20]\n"
+ "ldr q11, [%[outptr5], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x110]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr5], #0x20]\n"
+ "ldr q8, [%[outptr6], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x140]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr6], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+ default:
+ case 8:
+ {
+ if ((i+11) >= xmax)
+ {
+ for (int xi=0; xi<12; xi++)
+ {
+ if ((i+xi) < xmax)
+ {
+ *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta);
+ outptr0++;
+ *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta);
+ outptr1++;
+ *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta);
+ outptr2++;
+ *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta);
+ outptr3++;
+ *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta);
+ outptr4++;
+ *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta);
+ outptr5++;
+ *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta);
+ outptr6++;
+ *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta);
+ outptr7++;
+ }
+ }
+ inptr += 96;
+ } else {
+ /* Optimized routine to copy an entire block */
+ __asm __volatile (
+ "ldr q8, [%[outptr0]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr]]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0]]\n"
+ "ldr q9, [%[outptr1]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x30]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1]]\n"
+ "ldr q10, [%[outptr2]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x60]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2]]\n"
+ "ldr q11, [%[outptr3]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x90]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3]]\n"
+ "ldr q8, [%[outptr4]]\n"
+ "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0xc0]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4]]\n"
+ "ldr q9, [%[outptr5]]\n"
+ "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0xf0]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5]]\n"
+ "ldr q10, [%[outptr6]]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x120]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr6]]\n"
+ "ldr q11, [%[outptr7]]\n"
+ "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x150]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr7]]\n"
+ "ldr q8, [%[outptr0], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x10]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x10]\n"
+ "ldr q9, [%[outptr1], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x40]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x10]\n"
+ "ldr q10, [%[outptr2], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x70]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x10]\n"
+ "ldr q11, [%[outptr3], #0x10]\n"
+ "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0xa0]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x10]\n"
+ "ldr q8, [%[outptr4], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0xd0]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4], #0x10]\n"
+ "ldr q9, [%[outptr5], #0x10]\n"
+ "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x100]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5], #0x10]\n"
+ "ldr q10, [%[outptr6], #0x10]\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x130]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr6], #0x10]\n"
+ "ldr q11, [%[outptr7], #0x10]\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x160]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr7], #0x10]\n"
+ "ldr q8, [%[outptr0], #0x20]\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0x20]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr0], #0x20]\n"
+ "ldr q9, [%[outptr1], #0x20]\n"
+ "add %[outptr0], %[outptr0], #0x30\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x50]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr1], #0x20]\n"
+ "ldr q10, [%[outptr2], #0x20]\n"
+ "add %[outptr1], %[outptr1], #0x30\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x80]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr2], #0x20]\n"
+ "ldr q11, [%[outptr3], #0x20]\n"
+ "add %[outptr2], %[outptr2], #0x30\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0xb0]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr3], #0x20]\n"
+ "ldr q8, [%[outptr4], #0x20]\n"
+ "add %[outptr3], %[outptr3], #0x30\n"
+ "fmul v8.4s, v8.4s, %[beta].s[0]\n"
+ "ldr q4, [%[inptr], #0xe0]\n"
+ "fmla v8.4s, v4.4s, %[alpha].s[0]\n"
+ "str q8, [%[outptr4], #0x20]\n"
+ "ldr q9, [%[outptr5], #0x20]\n"
+ "add %[outptr4], %[outptr4], #0x30\n"
+ "fmul v9.4s, v9.4s, %[beta].s[0]\n"
+ "ldr q5, [%[inptr], #0x110]\n"
+ "fmla v9.4s, v5.4s, %[alpha].s[0]\n"
+ "str q9, [%[outptr5], #0x20]\n"
+ "ldr q10, [%[outptr6], #0x20]\n"
+ "add %[outptr5], %[outptr5], #0x30\n"
+ "fmul v10.4s, v10.4s, %[beta].s[0]\n"
+ "ldr q6, [%[inptr], #0x140]\n"
+ "fmla v10.4s, v6.4s, %[alpha].s[0]\n"
+ "str q10, [%[outptr6], #0x20]\n"
+ "ldr q11, [%[outptr7], #0x20]\n"
+ "add %[outptr6], %[outptr6], #0x30\n"
+ "fmul v11.4s, v11.4s, %[beta].s[0]\n"
+ "ldr q7, [%[inptr], #0x170]\n"
+ "fmla v11.4s, v7.4s, %[alpha].s[0]\n"
+ "str q11, [%[outptr7], #0x20]\n"
+ "add %[outptr7], %[outptr7], #0x30\n"
+ "add %[inptr], %[inptr], #0x180\n"
+ : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
+ [inptr] "+r" (inptr)
+ : [alpha] "w" (alpha), [beta] "w" (beta)
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"
+ );
+ }
+ }
+ break;
+
+
+ }
+ }
+ }
+ }
+}
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/transform.hpp b/src/core/NEON/kernels/arm_gemm/transform.hpp
index e422b91..0330783 100644
--- a/src/core/NEON/kernels/arm_gemm/transform.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transform.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -81,11 +81,14 @@
}
}
// "row" tail - row is out of range so fill with zeros always.
- for (int row = 0; row < blank_rows; row++) {
- for (int col=0; col < (fill_cols + blank_cols); col++) {
- *out++ = static_cast<TOut>(0);
- }
+ TOut zeroval = static_cast<TOut>(0);
+ int pads = blank_rows * (fill_cols + blank_cols);
+
+ for (int i=0; i<pads; i++) {
+ out[i] = zeroval;
}
+
+ out += pads;
}
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp
index 347eafb..0648ff6 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -23,7 +23,7 @@
*/
#pragma once
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE)
#include <arm_neon.h>
@@ -173,4 +173,4 @@
}
}
-#endif // __aarch64__
+#endif // __aarch64__ && !__ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
index fc1f2c2..e1ebba0 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -23,17 +23,14 @@
*/
#include "a32_interleave_6way_32bit.hpp"
#include "a32_transpose_interleave_8way_32bit.hpp"
-#ifdef __ARM_FEATURE_SVE
-#include "sve_interleave_8way_32bit.hpp"
-#include "sve_interleave_8way_block2_32bit.hpp"
-#include "sve_interleave_8way_block4_8bit.hpp"
-#else
-#include "a64_interleave_8way_32bit.hpp"
-#endif
#include "a64_block16_interleave4_8bit.hpp"
#include "a64_interleave_8way_16bit.hpp"
+#include "a64_interleave_8way_32bit.hpp"
#include "a64_interleave_8way_half_to_float.hpp"
#include "a64_transpose_interleave_12way_16bit.hpp"
#include "a64_transpose_interleave_12way_half_to_float.hpp"
#include "a64_transpose_interleave_24way_16bit.hpp"
-#include "transpose_interleave_common.hpp"
+#include "sve_interleave_8way_32bit.hpp"
+#include "sve_interleave_8way_block2_32bit.hpp"
+#include "sve_interleave_8way_block4_8bit.hpp"
+#include "transpose_interleave_common.hpp"
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
index 752e837..07c8219 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -41,7 +41,7 @@
long outpos = 0;
uint32_t *outptr = master_outptr;
- master_outptr += outwidth;
+ master_outptr += (outwidth * 1);
const uint32_t *inptr0 = inptr + y * ldin + k0;
const uint32_t *inptr1 = inptr0 + ldin;
@@ -60,52 +60,53 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
"incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
"zip2 z9.s, z0.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip1 z0.s, z8.s, z4.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z4.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z4.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z4.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "zip2 z11.s, z1.s, z4.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "zip1 z12.s, z2.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip2 z13.s, z2.s, z4.s\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip1 z14.s, z3.s, z4.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -115,60 +116,62 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "mov z14.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
"zip1 z8.s, z0.s, z4.s\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"zip2 z9.s, z0.s, z4.s\n"
+ "incw %[inpos], all, mul #1\n"
"zip1 z10.s, z1.s, z4.s\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip2 z11.s, z1.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip1 z0.s, z8.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z4.s\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "zip2 z3.s, z9.s, z4.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip1 z2.s, z9.s, z4.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "zip2 z3.s, z9.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "mov z14.s, #0\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip2 z5.s, z10.s, z14.s\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip1 z6.s, z11.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z7.s, z11.s, z14.s\n"
"zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z15.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -178,63 +181,66 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "mov z14.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"zip2 z9.s, z0.s, z4.s\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
"zip1 z10.s, z1.s, z4.s\n"
+ "incw %[inpos], all, mul #1\n"
"zip2 z11.s, z1.s, z4.s\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip1 z12.s, z2.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip2 z13.s, z2.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z4.s, z10.s, z14.s\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
+ "mov z14.s, #0\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip1 z6.s, z11.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z7.s, z11.s, z14.s\n"
"zip1 z8.s, z0.s, z4.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -244,65 +250,69 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
"zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
"zip1 z10.s, z1.s, z4.s\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
"zip2 z11.s, z1.s, z4.s\n"
+ "incw %[inpos], all, mul #1\n"
"zip1 z12.s, z2.s, z4.s\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip2 z13.s, z2.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip1 z14.s, z3.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
"zip2 z15.s, z3.s, z4.s\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -312,66 +322,71 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z5.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"incw %[inpos], all, mul #1\n"
"zip1 z10.s, z1.s, z5.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
"zip2 z11.s, z1.s, z5.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
"zip1 z12.s, z2.s, z5.s\n"
+ "ld1w z4.s, p0/z, [%[inptr4]]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip2 z13.s, z2.s, z5.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
"zip1 z14.s, z3.s, z5.s\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip2 z15.s, z3.s, z5.s\n"
+ "addvl %[inptr4], %[inptr4], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -381,67 +396,73 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z6.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"incw %[inpos], all, mul #1\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
"zip2 z13.s, z2.s, z6.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "ld1w z4.s, p0/z, [%[inptr4]]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "ld1w z5.s, p0/z, [%[inptr5]]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip1 z14.s, z3.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr4], %[inptr4], #1\n"
"zip2 z15.s, z3.s, z6.s\n"
+ "addvl %[inptr5], %[inptr5], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -451,68 +472,75 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z7.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"incw %[inpos], all, mul #1\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip1 z14.s, z3.s, z7.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "ld1w z4.s, p0/z, [%[inptr4]]\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[inptr5]]\n"
"zip2 z9.s, z0.s, z4.s\n"
+ "ld1w z6.s, p0/z, [%[inptr6]]\n"
"zip1 z10.s, z1.s, z5.s\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
"zip2 z11.s, z1.s, z5.s\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip1 z12.s, z2.s, z6.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr4], %[inptr4], #1\n"
"zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr5], %[inptr5], #1\n"
"zip2 z15.s, z3.s, z7.s\n"
+ "addvl %[inptr6], %[inptr6], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -522,69 +550,77 @@
"1:\n"
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
- "ld1w z7.s, p0/z, [%[inptr7], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
"incw %[inpos], all, mul #1\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
+ "ld1w z4.s, p0/z, [%[inptr4]]\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip1 z8.s, z0.s, z4.s\n"
+ "ld1w z5.s, p0/z, [%[inptr5]]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "ld1w z6.s, p0/z, [%[inptr6]]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "ld1w z7.s, p0/z, [%[inptr7]]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "addvl %[inptr4], %[inptr4], #1\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "addvl %[inptr5], %[inptr5], #1\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "addvl %[inptr6], %[inptr6], #1\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "addvl %[inptr7], %[inptr7], #1\n"
+ "zip2 z15.s, z3.s, z7.s\n"
"whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z3.s, z9.s, z13.s\n"
- "incw %[outpos], all, mul #1\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z5.s, z10.s, z14.s\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip1 z6.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip2 z7.s, z11.s, z15.s\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
diff --git a/src/core/NEON/kernels/arm_gemm/utils.hpp b/src/core/NEON/kernels/arm_gemm/utils.hpp
index a1fc00e..8b96c32 100644
--- a/src/core/NEON/kernels/arm_gemm/utils.hpp
+++ b/src/core/NEON/kernels/arm_gemm/utils.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -24,9 +24,7 @@
#pragma once
-#ifdef __ARM_FEATURE_SVE
-#include <arm_sve.h>
-#endif
+#include <cstddef>
// Macro for unreachable code (e.g. impossible default cases on switch)
#define UNREACHABLE(why) __builtin_unreachable()
@@ -49,13 +47,43 @@
}
}
+namespace arm_gemm {
+namespace utils {
+namespace {
+
+#ifdef __ARM_FEATURE_SVE
+template<size_t sz>
+inline unsigned long get_vector_length_sz() {
+ unsigned long v;
+
+ __asm (
+ "cntb %0"
+ : "=r" (v)
+ );
+
+ return v / sz;
+}
+
+#define VEC_LEN_SPEC(sz, opcode) template <> inline unsigned long get_vector_length_sz<sz>() { unsigned long v; __asm ( opcode " %0" : "=r" (v)); return v; }
+
+VEC_LEN_SPEC(8, "cntd")
+VEC_LEN_SPEC(4, "cntw")
+VEC_LEN_SPEC(2, "cnth")
+VEC_LEN_SPEC(1, "cntb")
+#endif
+
+} // anonymous namespace
+
template <typename T>
inline unsigned long get_vector_length() {
#ifdef __ARM_FEATURE_SVE
- const unsigned long length = svcntb();
+ return get_vector_length_sz<sizeof(T)>();
#else
- const unsigned long length = 16;
+ return 16 / sizeof(T);
#endif
+}
- return length / sizeof(T);
-}
\ No newline at end of file
+} // utils namespace
+} // arm_gemm namespace
+
+using namespace arm_gemm::utils;
\ No newline at end of file
diff --git a/src/core/NEON/kernels/assembly/Helpers.cpp b/src/core/NEON/kernels/assembly/Helpers.cpp
index 09ac08c..3d8d66d 100644
--- a/src/core/NEON/kernels/assembly/Helpers.cpp
+++ b/src/core/NEON/kernels/assembly/Helpers.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 ARM Limited.
+ * Copyright (c) 2018-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -24,91 +24,47 @@
#include "arm_compute/core/NEON/kernels/assembly/Helpers.h"
-#include "NEGEMMInterleavedStrategies.h"
+#include "arm_compute/core/NEON/kernels/assembly/arm_gemm.hpp"
namespace arm_compute
{
-namespace
-{
-template <typename InputType, bool use_dot = false>
-BlockSizes calculate_block_sizes_template(const CPUInfo &ci, unsigned int M, unsigned int N, unsigned int K)
-{
- using strategy = typename Kernel<InputType, use_dot>::strategy;
- return calculate_block_sizes<strategy>(ci, M, N, K);
-}
-} // namespace
-
-const char *get_strategy_name(DataType input_type, bool use_dot)
+arm_gemm::KernelDescription get_gemm_info(DataType input_type,
+ const CPUInfo &ci,
+ const unsigned int num_threads,
+ const INEGEMMWrapperKernel::Params &p,
+ float alpha,
+ float beta,
+ bool pretranspose_hint)
{
switch(input_type)
{
- case DataType::F32:
- return Kernel<float>::name;
#ifdef __aarch64__
- case DataType::U8:
case DataType::QASYMM8:
- if(use_dot)
- {
- return Kernel<uint8_t, true>::name;
- }
- else
- {
- return Kernel<uint8_t, false>::name;
- }
+ case DataType::U8:
+ {
+ arm_gemm::GemmArgs<uint32_t> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint);
+ return arm_gemm::get_gemm_method<uint8_t, uint32_t>(args);
+ }
case DataType::S8:
- if(use_dot)
- {
- return Kernel<int8_t, true>::name;
- }
- else
- {
- return Kernel<int8_t, false>::name;
- }
-#endif /* __aarch64__ */
+ {
+ arm_gemm::GemmArgs<int32_t> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint);
+ return arm_gemm::get_gemm_method<int8_t, int32_t>(args);
+ }
+#endif // __aarch64__
#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
case DataType::F16:
- return Kernel<__fp16>::name;
+ {
+ arm_gemm::GemmArgs<__fp16> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint);
+ return arm_gemm::get_gemm_method<__fp16, __fp16>(args);
+ }
#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
- default:
- ARM_COMPUTE_ERROR("DataType not supported");
- break;
- }
-}
-
-BlockSizes calculate_block_sizes_from_data_type(const CPUInfo &ci, unsigned int M, unsigned int N, unsigned int K, DataType input_type, bool use_dot)
-{
- switch(input_type)
- {
case DataType::F32:
- return calculate_block_sizes_template<float>(ci, M, N, K);
-#ifdef __aarch64__
- case DataType::U8:
- case DataType::QASYMM8:
- if(use_dot)
- {
- return calculate_block_sizes_template<uint8_t, true>(ci, M, N, K);
- }
- else
- {
- return calculate_block_sizes_template<uint8_t, false>(ci, M, N, K);
- }
- case DataType::S8:
- if(use_dot)
- {
- return calculate_block_sizes_template<int8_t, true>(ci, M, N, K);
- }
- else
- {
- return calculate_block_sizes_template<int8_t, false>(ci, M, N, K);
- }
-#endif /* __aarch64__ */
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
- case DataType::F16:
- return calculate_block_sizes_template<__fp16>(ci, M, N, K);
-#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
+ {
+ arm_gemm::GemmArgs<float> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint);
+ return arm_gemm::get_gemm_method<float, float>(args);
+ }
default:
- ARM_COMPUTE_ERROR("DataType not supported");
- break;
+ return arm_gemm::KernelDescription();
}
}
} // namespace arm_compute
diff --git a/src/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.cpp b/src/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.cpp
deleted file mode 100644
index 3b2975d..0000000
--- a/src/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.cpp
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Copyright (c) 2018 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.h"
-
-#include "NEGEMMInterleavedStrategies.h"
-#include "arm_compute/core/Helpers.h"
-#include "arm_compute/core/ITensor.h"
-#include "arm_compute/core/Utils.h"
-#include "arm_compute/core/Validate.h"
-#include "arm_compute/core/WindowIterator.h"
-
-namespace arm_compute
-{
-template <typename To, typename Tr, bool use_dot>
-void NEGEMMInterleavedMatrixMultiplyWrapperTemplate<To, Tr, use_dot>::configure(const ITensor *prepared_a, const ITensor *transformed_b, ITensor *tmp_c, ITensor *c, const Window &block_walker,
- const BlockSizes &block_sizes, const INEGEMMWrapperKernel::Params ¶ms, bool b_is_pretransposed, float alpha, float beta, unsigned int max_num_threads)
-{
- using strategy = typename Kernel<To, use_dot>::strategy;
-
- _prepared_a = prepared_a;
- _transformed_b = transformed_b;
- _tmp_c = tmp_c;
- _c = c;
- _block_walker = block_walker;
- _block_sizes = block_sizes;
- _params = params;
- _b_is_pretransposed = b_is_pretransposed;
- _alpha = alpha;
- _beta = beta;
-
- auto_init_if_empty(*_tmp_c->info(), c->info()->clone()->set_tensor_shape(TensorShape{ _block_sizes.x_block * strategy::out_height(), max_num_threads }));
-}
-
-template <typename To, typename Tr, bool use_dot>
-void NEGEMMInterleavedMatrixMultiplyWrapperTemplate<To, Tr, use_dot>::transform(const MatrixMultiplyWorkload &wl, const ThreadInfo &info, const Window &batch_window, const Coordinates &start_offset,
- const Coordinates &end_offset)
-{
- using strategy = typename Kernel<To, use_dot>::strategy;
-
- strategy strat(info.cpu_info);
- TensorAccessor<To> prepared_a(*_prepared_a);
- TensorAccessor<To> transformed_b(*_transformed_b);
- TensorAccessor<Tr> c(*_c);
- TensorAccessor<Tr> tmp_c(*_tmp_c);
-
- int prev_batch = -1;
- To *a_ptr = nullptr;
- auto window_iterator = arm_compute::create_window_iterator(batch_window, start_offset, end_offset, [&](const Coordinates & id)
- {
- const unsigned int y = id.x();
- const unsigned int batch = id.y();
- const unsigned int ymax = std::min(_params.M, y + strategy::out_height());
-
- // If it's the first block of a new batch then reset the pointer to A.
- if(prev_batch != static_cast<int>(batch))
- {
- const unsigned int first_m = id.x();
- a_ptr = prepared_a(0, first_m, batch);
- prev_batch = batch;
- }
-
- // Call matrix multiply assembly routine to process the block:
- strat.kernel(a_ptr, transformed_b(wl._offset_transformed_b), tmp_c(0, info.thread_id), 1, wl._bblocks, wl._kern_k);
- a_ptr += strategy::out_height() * wl._kern_k;
-
- // Merge the result with the other blocks' results:
- strat.transforms.Merge(c(0, 0, batch, wl._multi), tmp_c(0, info.thread_id), c.stride(1), y, ymax, wl._x0, wl._xmax, _alpha, (wl._k0 == 0 ? _beta : static_cast<Tr>(1)));
- });
- auto on_new_row_size = [&](unsigned int start, unsigned int end)
- {
- //Nothing to do
- };
- window_iterator.iterate_2D(on_new_row_size);
-}
-
-template <typename To, typename Tr, bool use_dot>
-void NEGEMMInterleavedMatrixMultiplyWrapperTemplate<To, Tr, use_dot>::create_workloads(std::vector<MatrixMultiplyWorkload> &workloads)
-{
- using strategy = typename Kernel<To, use_dot>::strategy;
-
- unsigned int offset_transformed_b = 0;
- unsigned int wl_index = 0;
- unsigned int num_buffers = 0, reshaped_block_size = 0;
-
- if(!_b_is_pretransposed)
- {
- num_buffers = _transformed_b->info()->tensor_shape()[1];
- reshaped_block_size = _transformed_b->info()->tensor_shape()[0];
- }
- execute_window_loop(_block_walker, [&](const Coordinates & id)
- {
- const unsigned int x0 = id.x();
- const unsigned int k0 = id.y();
- const unsigned int multi = id.z();
-
- const unsigned int xmax = std::min(x0 + _block_walker.x().step(), _params.N);
- const unsigned int kmax = std::min(k0 + _block_walker.y().step(), _params.K);
-
- // Figure out how many "K" the kernel will actually process.
- const int kern_k = ceil_to_multiple(kmax - k0, strategy::k_unroll());
- const int bblocks = DIV_CEIL(xmax - x0, strategy::out_width());
-
- workloads.push_back(MatrixMultiplyWorkload(offset_transformed_b, x0, xmax, k0, kmax, multi, kern_k, bblocks));
-
- if(_b_is_pretransposed)
- {
- offset_transformed_b += bblocks * strategy::out_width() * kern_k;
- }
- else
- {
- // Rotate through the BufferManager's buffers:
- wl_index++;
- offset_transformed_b = (wl_index % num_buffers) * reshaped_block_size;
- }
- });
-}
-
-//TODO: regroup somewhere ?
-template class NEGEMMInterleavedMatrixMultiplyWrapperTemplate<float, float>;
-#ifdef __aarch64__
-template class NEGEMMInterleavedMatrixMultiplyWrapperTemplate<uint8_t, uint32_t>;
-template class NEGEMMInterleavedMatrixMultiplyWrapperTemplate<int8_t, int32_t>;
-template class NEGEMMInterleavedMatrixMultiplyWrapperTemplate<uint8_t, uint32_t, true>;
-template class NEGEMMInterleavedMatrixMultiplyWrapperTemplate<int8_t, int32_t, true>;
-#endif /* __aarch64__ */
-
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-template class NEGEMMInterleavedMatrixMultiplyWrapperTemplate<float16_t, float16_t>;
-#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
-} // namespace arm_compute
diff --git a/src/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.cpp b/src/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.cpp
deleted file mode 100644
index 7fc57f3..0000000
--- a/src/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.cpp
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Copyright (c) 2018 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.h"
-
-#include "NEGEMMInterleavedStrategies.h"
-#include "arm_compute/core/Helpers.h"
-#include "arm_compute/core/ITensor.h"
-#include "arm_compute/core/Utils.h"
-#include "arm_compute/core/Validate.h"
-
-namespace arm_compute
-{
-namespace
-{
-// Call the lambda function for each workload generated by the passed window.
-template <typename To, bool use_dot, bool use_buffer_manager, typename Lambda>
-void for_each_element_in_window(const Window &window, const ITensor *b, ITensor *transformed_b, unsigned int N, unsigned int K, Lambda &&lambda)
-{
- using strategy = typename Kernel<To, use_dot>::strategy;
- unsigned int wl_index = 0;
- unsigned int num_buffers = 0, reshaped_block_size = 0;
-
- if(use_buffer_manager)
- {
- num_buffers = transformed_b->info()->tensor_shape()[1];
- reshaped_block_size = transformed_b->info()->strides_in_bytes().y();
- }
-
- unsigned int offset_transformed_b = transformed_b->info()->offset_first_element_in_bytes();
- execute_window_loop(window, [&](const Coordinates & coordinates)
- {
- const unsigned int x0 = coordinates.x();
- const unsigned int k0 = coordinates.y();
- const unsigned int multi = coordinates.z();
-
- const unsigned int offset_b = b->info()->offset_element_in_bytes(Coordinates(0, 0, multi));
- const unsigned int xmax = std::min(x0 + window.x().step(), N);
- const unsigned int kmax = std::min(k0 + window.y().step(), K);
-
- /* Figure out the size of each block. */
- unsigned int x_size = (xmax - x0);
- unsigned int k_size = (kmax - k0);
-
- /* Round sizes up as needed. */
- x_size = ceil_to_multiple(x_size, strategy::out_width());
- k_size = ceil_to_multiple(k_size, strategy::k_unroll());
-
- lambda(PrepareBWorkload(offset_b, offset_transformed_b, x0, xmax, k0, kmax));
-
- //Each workload represents one block:
- if(use_buffer_manager)
- {
- // Rotate through the BufferManager's buffers:
- wl_index++;
- offset_transformed_b = (wl_index % num_buffers) * reshaped_block_size;
- }
- else
- {
- offset_transformed_b += (x_size * k_size * sizeof(To));
- }
- });
-}
-
-// Calculate the size of transformed_b:
-template <typename To, bool use_dot>
-unsigned int get_B_pretransposed_array_size(unsigned int N, unsigned int K, const BlockSizes &bs, unsigned int multis)
-{
- using strategy = typename Kernel<To, use_dot>::strategy;
-
- // How many full blocks do N / K contain ?
- size_t num_full_k = K / bs.k_block;
- size_t num_full_x = N / bs.x_block;
-
- ARM_COMPUTE_ERROR_ON(bs.x_block % strategy::out_width() != 0);
- ARM_COMPUTE_ERROR_ON(bs.k_block % strategy::k_unroll() != 0);
-
- size_t normal_x_size = bs.x_block;
- size_t normal_k_size = bs.k_block;
-
- // Round up the leftovers to be a multiple of the strategy processing size:
- size_t left_over_x_size = ceil_to_multiple(N % bs.x_block, strategy::out_width());
- size_t left_over_k_size = ceil_to_multiple(K % bs.k_block, strategy::k_unroll());
-
- // Calculate the total size of the buffer:
- size_t total = num_full_k * normal_k_size * (num_full_x * normal_x_size + left_over_x_size);
- total += left_over_k_size * (left_over_x_size + num_full_x * normal_x_size);
-
- total *= multis;
-
- return total;
-}
-
-} // namespace
-
-template <typename To, bool use_dot>
-BlockSizes NEGEMMInterleavedPrepareBWrapperKernelTemplate<To, use_dot>::block_sizes() const
-{
- return _block_sizes;
-}
-
-template <typename To, bool use_dot>
-void NEGEMMInterleavedPrepareBWrapperKernelTemplate<To, use_dot>::configure(const ITensor *b, ITensor *transformed_b, bool transpose_b, const CPUInfo &ci, const INEGEMMWrapperKernel::Params ¶ms)
-{
- using strategy = typename Kernel<To, use_dot>::strategy;
-
- const unsigned int multis = b->info()->tensor_shape().z();
- _Nsize = b->info()->tensor_shape().x();
- _Ksize = b->info()->tensor_shape().y();
- _b = b;
- _transformed_b = transformed_b;
- _transpose_b = transpose_b;
-
- _block_sizes = calculate_block_sizes<strategy>(ci, params.M, params.N, params.K);
-
- auto_init_if_empty(*transformed_b->info(), b->info()->clone()->set_tensor_shape(TensorShape{ get_B_pretransposed_array_size<To, use_dot>(_Nsize, _Ksize, _block_sizes, multis) }));
-
- Window window;
- window.set(Window::DimX, Window::Dimension(0, ceil_to_multiple(_Nsize, _block_sizes.x_block), _block_sizes.x_block));
- window.set(Window::DimY, Window::Dimension(0, ceil_to_multiple(_Ksize, _block_sizes.k_block), _block_sizes.k_block));
- window.set(Window::DimZ, Window::Dimension(0, multis));
-
- INEKernel::configure(window);
-}
-
-template <typename To, bool use_dot>
-void NEGEMMInterleavedPrepareBWrapperKernelTemplate<To, use_dot>::transform(const PrepareBWorkload &wl, const ThreadInfo &info)
-{
- using strategy = typename Kernel<To, use_dot>::strategy;
-
- strategy strat(info.cpu_info);
- strat.transforms.PrepareB(reinterpret_cast<To *>(_transformed_b->buffer() + wl._offset_transformed_b),
- reinterpret_cast<To *>(_b->buffer() + wl._offset_b),
- _b->info()->strides_in_bytes().y() / sizeof(To),
- wl._x0, wl._xmax, wl._k0, wl._kmax, _transpose_b);
-}
-
-template <typename To, bool use_dot>
-void NEGEMMInterleavedPrepareBWrapperKernelTemplate<To, use_dot>::create_workloads(std::vector<PrepareBWorkload> &workloads)
-{
- for_each_element_in_window<To, use_dot, true>(window(), _b, _transformed_b, _Nsize, _Ksize, [&workloads](PrepareBWorkload && wl)
- {
- workloads.push_back(std::move(wl));
- });
-}
-
-template <typename To, bool use_dot>
-void NEGEMMInterleavedPrepareBWrapperKernelTemplate<To, use_dot>::run(const Window &window, const ThreadInfo &info)
-{
- ARM_COMPUTE_ERROR_ON_MISMATCHING_WINDOWS(window, INEKernel::window());
- for_each_element_in_window<To, use_dot, false>(window, _b, _transformed_b, _Nsize, _Ksize, [&](PrepareBWorkload && wl)
- {
- this->transform(wl, info);
- });
-}
-
-template class NEGEMMInterleavedPrepareBWrapperKernelTemplate<float>;
-#ifdef __aarch64__
-template class NEGEMMInterleavedPrepareBWrapperKernelTemplate<uint8_t>;
-template class NEGEMMInterleavedPrepareBWrapperKernelTemplate<int8_t>;
-template class NEGEMMInterleavedPrepareBWrapperKernelTemplate<uint8_t, true>;
-template class NEGEMMInterleavedPrepareBWrapperKernelTemplate<int8_t, true>;
-#endif /* __aarch64__ */
-
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-template class NEGEMMInterleavedPrepareBWrapperKernelTemplate<float16_t>;
-#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
-} // namespace arm_compute
diff --git a/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h b/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h
index 69842fe..da6ef2d 100644
--- a/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h
+++ b/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 ARM Limited.
+ * Copyright (c) 2018-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -44,90 +44,175 @@
namespace arm_compute
{
-namespace
+namespace detail
{
-template <typename To, bool use_dot = false>
-struct Kernel
+/** GEMM Interleaved Strategy interface */
+class IInterleavedStrategy
{
+public:
+ /** Virtual Destructor */
+ virtual ~IInterleavedStrategy() = default;
+ /** Instantiate and configure a prepareB Kernel
+ *
+ * @param[in] b Input tensor B.
+ * @param[in] transformed_b Reshaped tensor B.
+ * @param[in] params GM, N, K sizes.
+ * @param[in] ci CPUInfo to be used for kernel configuration.
+ *
+ * @return A wrapped specialized prepareB kernel
+ */
+ virtual std::unique_ptr<NEGEMMInterleavedPrepareBWrapperKernel> instantiate_prepareB(const ITensor *b,
+ ITensor *transformed_b,
+ const INEGEMMWrapperKernel::Params ¶ms,
+ const CPUInfo &ci) = 0;
+ /** Instantiate and configure a transformA Kernel
+ *
+ * @param[in] a Input tensor A.
+ * @param[in] transformed_a Reshaped tensor A.
+ * @param[in] block_walker Window representing the layout of the matrix's blocks.
+ * @param[in] params M, N, K sizes.
+ *
+ * @return A wrapped specialized transformA kernel
+ */
+ virtual std::unique_ptr<NEGEMMInterleavedTransformAWrapper> instantiate_transformA(const ITensor *a,
+ ITensor *transformed_a,
+ const Window &block_walker,
+ const INEGEMMWrapperKernel::Params ¶ms) = 0;
+ /** Instantiate and configure a prepareB Kernel
+ *
+ * @param transformed_a Already reshaped tensor A.
+ * @param transformed_b Already reshaped tensor B.
+ * @param tmp_c Temporary buffer to be used to store intermediate results.
+ * @param c Result tensor C.
+ * @param block_walker Window containing iteration information for the M and batch dimensions.
+ * @param block_sizes Block sizes to use for the matrix multiplication (A & B must have been reshaped using these same block sizes).
+ * @param params M, N, K sizes.
+ * @param alpha Alpha value
+ * @param beta Beta value
+ * @param pretranspose_b Is B also pretransposed ?
+ * @param num_threads Maximum number of threads that might be used for the calculations.
+ *
+ * @return A wrapped specialized MatrixMultiply kernel
+ */
+ virtual std::unique_ptr<NEGEMMInterleavedMatrixMultiplyWrapper> instantiate_matrix_multiply(const ITensor *transformed_a, const ITensor *transformed_b, ITensor *tmp_c, ITensor *c,
+ const Window &block_walker, const BlockSizes &block_sizes,
+ const INEGEMMWrapperKernel::Params ¶ms, float alpha, float beta, bool pretranspose_b,
+ unsigned int num_threads) = 0;
+ /** Calculates the block sizes of a given strategy
+ *
+ * @param[in] ci CPUInfo to be used for kernel configuration.
+ * @param[in] params M, N, K sizes.
+ *
+ * @return BlockSizes for a given strategy
+ */
+ virtual BlockSizes calculate_block_sizes_for_strategy(const CPUInfo &ci, const INEGEMMWrapperKernel::Params ¶ms) = 0;
};
-#define DEFINE_STRATEGY_SUFFIX(strat, suffix) \
- using strategy = arm_gemm::strat; \
- static constexpr const char *name = #strat suffix;
+/** Interleaved Strategy class */
+template <typename StrategyType>
+class InterleavedStrategy : public IInterleavedStrategy
+{
+public:
+ using strategy = StrategyType;
-#define DEFINE_STRATEGY(strat) \
- DEFINE_STRATEGY_SUFFIX(strat, "")
+public:
+ // Inherited methods overridden
+ std::unique_ptr<NEGEMMInterleavedPrepareBWrapperKernel> instantiate_prepareB(const ITensor *b,
+ ITensor *transformed_b,
+ const INEGEMMWrapperKernel::Params ¶ms,
+ const CPUInfo &ci) override
+ {
+ auto prepare_b = support::cpp14::make_unique<NEGEMMInterleavedPrepareBWrapperKernelTemplate<strategy>>();
+ prepare_b->configure(b, transformed_b, false, ci, params);
+ return std::move(prepare_b);
+ }
+ std::unique_ptr<NEGEMMInterleavedTransformAWrapper> instantiate_transformA(const ITensor *a,
+ ITensor *transformed_a,
+ const Window &block_walker,
+ const INEGEMMWrapperKernel::Params ¶ms) override
+ {
+ auto transform_a = support::cpp14::make_unique<NEGEMMInterleavedTransformAWrapperTemplate<strategy>>();
+ transform_a->configure(a, transformed_a, false, block_walker, params);
+ return std::move(transform_a);
+ }
+ std::unique_ptr<NEGEMMInterleavedMatrixMultiplyWrapper> instantiate_matrix_multiply(const ITensor *transformed_a, const ITensor *transformed_b, ITensor *tmp_c, ITensor *c,
+ const Window &block_walker, const BlockSizes &block_sizes,
+ const INEGEMMWrapperKernel::Params ¶ms, float alpha, float beta, bool pretranspose_b,
+ unsigned int num_threads) override
+ {
+ auto matrix_multiply = support::cpp14::make_unique<NEGEMMInterleavedMatrixMultiplyWrapperTemplate<strategy>>();
+ matrix_multiply->configure(transformed_a, transformed_b, tmp_c, c, block_walker, block_sizes, params, pretranspose_b, alpha, beta, num_threads);
+ return std::move(matrix_multiply);
+ }
-#ifdef __ARM_FEATURE_SVE
-template <>
-struct Kernel<float, false>
-{
- DEFINE_STRATEGY(interleaved_fp32_mla_3VLx8)
-};
-template <>
-struct Kernel<float16_t, false>
-{
- DEFINE_STRATEGY(interleaved_fp16_mla_3VLx8)
-};
-template <bool use_dot>
-struct Kernel<int8_t, use_dot>
-{
- DEFINE_STRATEGY(interleaved_s8s32_dot_3VLx8)
-};
-template <bool use_dot>
-struct Kernel<uint8_t, use_dot>
-{
- DEFINE_STRATEGY(interleaved_u8u32_dot_3VLx8)
-};
-#else /* __ARM_FEATURE_SVE */
-
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-template <>
-struct Kernel<float16_t, false>
-{
- DEFINE_STRATEGY(hgemm_24x8)
-};
-#endif /*__ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
-#ifdef __aarch64__
-template <>
-struct Kernel<float, false>
-{
- DEFINE_STRATEGY(sgemm_12x8)
-};
-template <>
-struct Kernel<int8_t, false>
-{
- DEFINE_STRATEGY(gemm_s8_4x4)
-};
-template <>
-struct Kernel<uint8_t, false>
-{
- DEFINE_STRATEGY(gemm_u8_4x4)
+ BlockSizes calculate_block_sizes_for_strategy(const CPUInfo &ci, const INEGEMMWrapperKernel::Params ¶ms) override
+ {
+ return calculate_block_sizes<strategy>(ci, params.M, params.N, params.K);
+ }
};
-//Use different strategies for 8bit dot product:
-template <>
-struct Kernel<int8_t, true>
+/** Create the backend GEMM strategy to use given the provided kernel info
+ *
+ * @param[in] kernel_name Kernel name of the backend strategy to instantiate
+ *
+ * @return The requested kernel strategy if exists else nullptr
+ */
+std::unique_ptr<IInterleavedStrategy> create_strategy(const std::string &kernel_name)
{
- DEFINE_STRATEGY_SUFFIX(gemm_s8_12x8, "_dot")
-};
-template <>
-struct Kernel<uint8_t, true>
-{
- DEFINE_STRATEGY_SUFFIX(gemm_u8_12x8, "_dot")
-};
-#else
-template <>
-struct Kernel<float, false>
-{
- DEFINE_STRATEGY(sgemm_8x6)
-};
-#endif /* __aarch64__ */
-#endif /* __ARM_FEATURE_SVE */
-
-#undef DEFINE_STRATEGY
-#undef DEFINE_STRATEGY_SUFFIX
-
-} // namespace
+#if defined(__arm__)
+ if(kernel_name.find("sgemm_8x6") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::sgemm_8x6>>();
+ }
+#endif // defined(__arm__)
+#if defined(__aarch64__)
+ if(kernel_name.find("gemm_s8_4x4") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::gemm_s8_4x4>>();
+ }
+ if(kernel_name.find("gemm_s8_12x8") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::gemm_s8_12x8>>();
+ }
+ if(kernel_name.find("gemm_u8_4x4") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::gemm_u8_4x4>>();
+ }
+ if(kernel_name.find("gemm_u8_12x8") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::gemm_u8_12x8>>();
+ }
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+ if(kernel_name.find("hgemm_24x8") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::hgemm_24x8>>();
+ }
+#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+ if(kernel_name.find("sgemm_12x8") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::sgemm_12x8>>();
+ }
+#if defined(__ARM_FEATURE_SVE)
+ if(kernel_name.find("interleaved_fp16_mla_3VLx8") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::interleaved_fp16_mla_3VLx8>>();
+ }
+ if(kernel_name.find("interleaved_fp32_mla_3VLx8") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::interleaved_fp32_mla_3VLx8>>();
+ }
+ if(kernel_name.find("interleaved_s8s32_dot_3VLx8") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::interleaved_s8s32_dot_3VLx8>>();
+ }
+ if(kernel_name.find("interleaved_u8u32_dot_3VLx8") != std::string::npos)
+ {
+ return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::interleaved_u8u32_dot_3VLx8>>();
+ }
+#endif // defined(__ARM_FEATURE_SVE)
+#endif // defined(__aarch64__)_
+ return nullptr;
+}
+} // namespace detail
} // namespace arm_compute
#endif /* __ARM_COMPUTE_NEGEMMINTERLEAVEDSTRATEGIES_H__ */
diff --git a/src/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.cpp b/src/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.cpp
deleted file mode 100644
index 3b80a1f..0000000
--- a/src/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.cpp
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Copyright (c) 2018 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.h"
-
-#include "NEGEMMInterleavedStrategies.h"
-#include "arm_compute/core/Helpers.h"
-#include "arm_compute/core/ITensor.h"
-#include "arm_compute/core/Utils.h"
-#include "arm_compute/core/Validate.h"
-#include "arm_compute/core/WindowIterator.h"
-
-#include "utils/TypePrinter.h"
-
-namespace arm_compute
-{
-template <typename To, bool use_dot>
-void NEGEMMInterleavedTransformAWrapperTemplate<To, use_dot>::configure(const ITensor *a, ITensor *transformed_a, bool transpose_a, const Window &block_walker,
- const INEGEMMWrapperKernel::Params ¶ms)
-{
- _a = a;
- _transformed_a = transformed_a;
- _transpose_a = transpose_a;
- _Ksize = params.K;
- _Msize = params.M;
- _k_multi_window = block_walker.shift_dimensions(1); // block_walker contains (M,K,Multi) --> shift by 1 to get rid of the "M" dimension
-}
-
-template <typename To, bool use_dot>
-void NEGEMMInterleavedTransformAWrapperTemplate<To, use_dot>::transform(const TransformAWorkload &wl, const ThreadInfo &info, const Window &batch_window, const Coordinates &start_offset,
- const Coordinates &end_offset)
-{
- using strategy = typename Kernel<To, use_dot>::strategy;
-
- strategy strat(info.cpu_info);
- TensorAccessor<To> a(*_a);
- TensorAccessor<To> transformed_a(*_transformed_a);
-
- if(_a->info()->data_layout() == DataLayout::NHWC)
- {
- // In the case of NHWC we want to interpret the output shape as 3D. Thus, the batch stride for A is
- // the relevant multiple of the row stride.
- const size_t nhwc_batch_stride = _a->info()->strides_in_bytes().y() * _Msize;
- a.set_stride(2, nhwc_batch_stride);
- }
-
- unsigned int last_m = 0;
- //TODO: Create a new iterate_1D( DimY);
- int last_y = -1;
- auto window_iterator = arm_compute::create_window_iterator(batch_window, start_offset, end_offset, [&](const Coordinates & id)
- {
- if(id.y() != last_y)
- {
- last_y = id.y();
- unsigned int batch = id.y();
- unsigned int first_m = id.x();
-
- if(first_m >= last_m)
- return;
-
- strat.transforms.PrepareA(transformed_a(0, first_m, batch),
- a(0, 0, batch, wl._multi),
- a.stride(1), first_m, last_m, wl._k0, wl._kmax, _transpose_a);
- }
- });
- auto on_new_row_size = [&](unsigned int start, unsigned int end)
- {
- last_m = std::min(end, _Msize);
- };
- window_iterator.iterate_2D(on_new_row_size);
-}
-
-template <typename To, bool use_dot>
-void NEGEMMInterleavedTransformAWrapperTemplate<To, use_dot>::create_workloads(std::vector<TransformAWorkload> &workloads)
-{
- execute_window_loop(_k_multi_window, [&](const Coordinates & id)
- {
- const unsigned int k0 = id.x();
- const unsigned int multi = id.y();
- const unsigned int kmax = std::min(k0 + _k_multi_window.x().step(), _Ksize);
-
- workloads.push_back(TransformAWorkload(k0, kmax, multi));
- });
-}
-
-template class NEGEMMInterleavedTransformAWrapperTemplate<float>;
-#ifdef __aarch64__
-template class NEGEMMInterleavedTransformAWrapperTemplate<uint8_t>;
-template class NEGEMMInterleavedTransformAWrapperTemplate<int8_t>;
-template class NEGEMMInterleavedTransformAWrapperTemplate<uint8_t, true>;
-template class NEGEMMInterleavedTransformAWrapperTemplate<int8_t, true>;
-#endif /* __aarch64__ */
-
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-template class NEGEMMInterleavedTransformAWrapperTemplate<float16_t>;
-#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
-} // namespace arm_compute
diff --git a/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp b/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp
index e452dfb..7b1f3e7 100644
--- a/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp
+++ b/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 ARM Limited.
+ * Copyright (c) 2018-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -34,11 +34,7 @@
#include "../arm_gemm/mergeresults.hpp"
#include "../arm_gemm/transform.hpp"
-#include "../arm_gemm/kernels/a32_sgemm_8x6.hpp"
-#include "../arm_gemm/kernels/a64_sgemm_12x8.hpp"
#include "../arm_gemm/kernels/a64_sgemm_native_16x4.hpp"
-#include "../arm_gemm/kernels/a64_sgemv_pretransposed.hpp"
-#include "../arm_gemm/kernels/a64_sgemv_trans.hpp"
namespace arm_compute
{
diff --git a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp
index 25be4a5..cd614ba 100644
--- a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp
+++ b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 ARM Limited.
+ * Copyright (c) 2018-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -24,9 +24,6 @@
#include "arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h"
#include "arm_compute/core/CPP/Validate.h"
-#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.h"
-#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.h"
-#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.h"
#include "arm_compute/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.h"
#include "arm_compute/runtime/NEON/NEScheduler.h"
#include "arm_compute/runtime/NEON/functions/NESimpleAssemblyFunction.h"
@@ -38,14 +35,14 @@
{
namespace
{
-std::unique_ptr<IFunction> create_function_all_types(arm_gemm::GemmMethod method, const ITensor *a, const ITensor *b, ITensor *d, float alpha, float beta, bool pretranspose_hint,
+std::unique_ptr<IFunction> create_function_all_types(arm_gemm::KernelDescription gemm_kernel_info,
+ const ITensor *a, const ITensor *b, ITensor *d, float alpha, float beta, bool pretranspose_hint,
std::shared_ptr<IMemoryManager> memory_manager)
{
//Note: It's safe to not check for FP16 support because this was already checked in NEGEMMAssemblyDispatch::configure()
- switch(method)
+ switch(gemm_kernel_info.method)
{
- case arm_gemm::GemmMethod::GEMM_INTERLEAVED_FP16:
case arm_gemm::GemmMethod::GEMM_INTERLEAVED:
{
if(!pretranspose_hint)
@@ -56,92 +53,24 @@
function->configure(a, b, d, alpha, beta, pretranspose_hint);
return std::move(function);
}
- default:
- return nullptr;
- }
-}
-
-template <typename TypeInput, typename TypeOutput>
-std::unique_ptr<IFunction> create_function(arm_gemm::GemmMethod method, const ITensor *a, const ITensor *b, ITensor *d, float alpha, float beta, bool pretranspose_hint,
- std::shared_ptr<IMemoryManager> memory_manager)
-{
- ARM_COMPUTE_UNUSED(method);
- ARM_COMPUTE_UNUSED(a);
- ARM_COMPUTE_UNUSED(b);
- ARM_COMPUTE_UNUSED(d);
- ARM_COMPUTE_UNUSED(alpha);
- ARM_COMPUTE_UNUSED(beta);
- ARM_COMPUTE_UNUSED(pretranspose_hint);
- ARM_COMPUTE_UNUSED(memory_manager);
- return nullptr;
-}
-
-#ifdef __aarch64__
-template <>
-std::unique_ptr<IFunction> create_function<int8_t, int32_t>(arm_gemm::GemmMethod method, const ITensor *a, const ITensor *b, ITensor *d, float alpha, float beta, bool pretranspose_hint,
- std::shared_ptr<IMemoryManager> memory_manager)
-{
- switch(method)
- {
- case arm_gemm::GemmMethod::GEMM_INTERLEAVED_DOT:
- {
- if(!pretranspose_hint)
- {
- return nullptr;
- }
- auto function = support::cpp14::make_unique<NEGEMMInterleavedWrapper>(memory_manager);
- function->configure(a, b, d, alpha, beta, pretranspose_hint, true /* use_dot */);
- return std::move(function);
- }
- default:
- return nullptr;
- }
- return nullptr;
-}
-
-template <>
-std::unique_ptr<IFunction> create_function<uint8_t, uint32_t>(arm_gemm::GemmMethod method, const ITensor *a, const ITensor *b, ITensor *d, float alpha, float beta, bool pretranspose_hint,
- std::shared_ptr<IMemoryManager> memory_manager)
-{
- switch(method)
- {
- case arm_gemm::GemmMethod::GEMM_INTERLEAVED_DOT:
- {
- if(!pretranspose_hint)
- {
- return nullptr;
- }
- auto function = support::cpp14::make_unique<NEGEMMInterleavedWrapper>(memory_manager);
- function->configure(a, b, d, alpha, beta, pretranspose_hint, true /* use_dot */);
- return std::move(function);
- }
- default:
- return nullptr;
- }
- return nullptr;
-}
-
-template <>
-std::unique_ptr<IFunction> create_function<float, float>(arm_gemm::GemmMethod method, const ITensor *a, const ITensor *b, ITensor *d, float alpha, float beta, bool pretranspose_hint,
- std::shared_ptr<IMemoryManager> memory_manager)
-{
- ARM_COMPUTE_UNUSED(pretranspose_hint);
- ARM_COMPUTE_UNUSED(memory_manager);
- switch(method)
- {
+#if defined(__aarch64__)
case arm_gemm::GemmMethod::GEMM_NATIVE:
{
- auto kernel = support::cpp14::make_unique<NEGEMMNativeWrapperKernel<float, float>>();
- kernel->configure(a, b, d, alpha, beta);
- auto function = support::cpp14::make_unique<NESimpleAssemblyFunction>();
- function->configure(std::move(kernel));
- return std::move(function);
+ if(gemm_kernel_info.name.find("sgemm_native_16x4") != std::string::npos)
+ {
+ auto kernel = support::cpp14::make_unique<NEGEMMNativeWrapperKernel<float, float>>();
+ kernel->configure(a, b, d, alpha, beta);
+ auto function = support::cpp14::make_unique<NESimpleAssemblyFunction>();
+ function->configure(std::move(kernel));
+ return std::move(function);
+ }
+ return nullptr;
}
+#endif // defined(__aarch64__)
default:
return nullptr;
}
}
-#endif /* __aarch64__ */
/** Fallback in case ACL doesn't have a function */
template <typename TypeInput, typename TypeOutput>
@@ -189,7 +118,7 @@
template <typename TypeInput, typename TypeOutput>
void Fallback<TypeInput, TypeOutput>::configure(const ITensor *a, const ITensor *b, ITensor *d, arm_gemm::GemmArgs<TypeOutput> &args, MemoryGroup &memory_group)
{
- _gemm_kernel_asm = arm_gemm::gemm<TypeInput, TypeOutput>(args, nullptr);
+ _gemm_kernel_asm = arm_gemm::gemm<TypeInput, TypeOutput>(args);
if(_gemm_kernel_asm == nullptr)
{
//configuration not supported: Leave function unconfigured:
@@ -334,12 +263,8 @@
arm_gemm::GemmArgs<TypeOutput> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint);
//Try to create an ACL function:
- acl_function = create_function_all_types(arm_gemm::get_gemm_method<TypeInput, TypeOutput>(args), a, b, d, alpha, beta, pretranspose_hint, memory_manager);
- // If the type agnostic factory failed to create an ACL function, try the specialised one:
- if(acl_function == nullptr)
- {
- acl_function = create_function<TypeInput, TypeOutput>(arm_gemm::get_gemm_method<TypeInput, TypeOutput>(args), a, b, d, alpha, beta, pretranspose_hint, memory_manager);
- }
+ acl_function = create_function_all_types(arm_gemm::get_gemm_method<TypeInput, TypeOutput>(args), a, b, d, alpha, beta, pretranspose_hint, std::move(memory_manager));
+
//If we still don't have an ACL function:
if(acl_function == nullptr)
{
diff --git a/src/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.cpp b/src/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.cpp
index fe998a0..695fc85 100644
--- a/src/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.cpp
+++ b/src/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 ARM Limited.
+ * Copyright (c) 2018-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -26,12 +26,11 @@
#include "arm_compute/core/ITensor.h"
#include "arm_compute/core/NEON/kernels/assembly/Helpers.h"
-#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.h"
-#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.h"
-#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.h"
#include "arm_compute/core/Utils.h"
#include "arm_compute/runtime/NEON/NEScheduler.h"
+#include "src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h"
+
#include <atomic>
#include <condition_variable>
#include <mutex>
@@ -179,6 +178,7 @@
: _memory_group(std::move(memory_manager))
{
}
+
void NEGEMMInterleavedWrapper::run()
{
prepare();
@@ -334,38 +334,7 @@
}
}
-namespace
-{
-// Factory to instantiate NEGEMMInterleavedPrepareBWrapperKernel:
-template <typename InputType, bool use_dot = false>
-std::unique_ptr<NEGEMMInterleavedPrepareBWrapperKernel> instantiate_prepareB(const ITensor *b, ITensor *transformed_b, const INEGEMMWrapperKernel::Params ¶ms)
-{
- auto prepare_b = support::cpp14::make_unique<NEGEMMInterleavedPrepareBWrapperKernelTemplate<InputType, use_dot>>();
- prepare_b->configure(b, transformed_b, false, NEScheduler::get().cpu_info(), params);
- return std::move(prepare_b);
-}
-
-// Factory to instantiate NEGEMMInterleavedTransformAWrapperTemplate:
-template <typename InputType, bool use_dot = false>
-std::unique_ptr<NEGEMMInterleavedTransformAWrapper> instantiate_transformA(const ITensor *a, ITensor *transformed_a, const Window &block_walker, const INEGEMMWrapperKernel::Params ¶ms)
-{
- auto transform_a = support::cpp14::make_unique<NEGEMMInterleavedTransformAWrapperTemplate<InputType, use_dot>>();
- transform_a->configure(a, transformed_a, false, block_walker, params);
- return std::move(transform_a);
-}
-
-// Factory to instantiate NEGEMMInterleavedTransformAWrapperTemplate:
-template <typename InputType, typename OutputType, bool use_dot = false>
-std::unique_ptr<NEGEMMInterleavedMatrixMultiplyWrapper> instantiate_matrix_multiply(const ITensor *transformed_a, const ITensor *transformed_b, ITensor *tmp_c, ITensor *c, const Window &block_walker,
- const BlockSizes &block_sizes, const INEGEMMWrapperKernel::Params ¶ms, bool pretranspose_b, float alpha, float beta)
-{
- auto matrix_multiply = support::cpp14::make_unique<NEGEMMInterleavedMatrixMultiplyWrapperTemplate<InputType, OutputType, use_dot>>();
- matrix_multiply->configure(transformed_a, transformed_b, tmp_c, c, block_walker, block_sizes, params, pretranspose_b, alpha, beta, NEScheduler::get().num_threads());
- return std::move(matrix_multiply);
-}
-} // namespace
-
-void NEGEMMInterleavedWrapper::configure(const ITensor *a, const ITensor *b, ITensor *c, float alpha, float beta, bool pretranspose_b, bool use_dot)
+void NEGEMMInterleavedWrapper::configure(const ITensor *a, const ITensor *b, ITensor *c, float alpha, float beta, bool pretranspose_b)
{
_params = INEGEMMWrapperKernel::extract_parameters(a, b, c);
_a = a;
@@ -373,18 +342,26 @@
_c = c;
_pretranspose_b = pretranspose_b;
- DataType input_type = a->info()->data_type();
+ const DataType input_type = a->info()->data_type();
+ const CPUInfo &ci = NEScheduler::get().cpu_info();
+ const unsigned int num_threads = NEScheduler::get().num_threads();
+
+ const arm_gemm::KernelDescription gemm_kernel_info = get_gemm_info(input_type, ci, num_threads, _params, alpha, beta, pretranspose_b);
+ ARM_COMPUTE_ERROR_ON(gemm_kernel_info.method != arm_gemm::GemmMethod::GEMM_INTERLEAVED);
// Forcing 128-byte alignment (required by 32-bit kernels)
const unsigned int alignment = 128;
_transformed_b.allocator()->init(TensorInfo{}, alignment);
_tmp_c.allocator()->init(TensorInfo{}, alignment);
- _tag = "NEGEMMInterleaved_";
- _tag += get_strategy_name(input_type, use_dot);
+ _tag = "NEGEMMInterleaved_" + gemm_kernel_info.name;
+
+ // Get strategy
+ std::unique_ptr<detail::IInterleavedStrategy> strategy = detail::create_strategy(gemm_kernel_info.name);
+ ARM_COMPUTE_ERROR_ON(strategy == nullptr);
if(!_pretranspose_b)
{
- _block_sizes = calculate_block_sizes_from_data_type(NEScheduler::get().cpu_info(), _params.M, _params.N, _params.K, input_type, use_dot);
+ _block_sizes = strategy->calculate_block_sizes_for_strategy(ci, _params);
_batch_window.set(Window::DimX, Window::Dimension(0, ceil_to_multiple(_block_sizes.m_round, _block_sizes.strategy_out_height), _block_sizes.strategy_out_height));
_batch_window.set(Window::DimY, Window::Dimension(0, _params.batches));
// If the execution is single threaded or has only one window then the buffer manager only needs 1 buffer else we will use NUM_BUFFERS buffers and ping pong between them:
@@ -409,43 +386,8 @@
{
_tag += "_preB";
}
- switch(input_type)
- {
- case DataType::F32:
- _prepare_b = instantiate_prepareB<float>(_b, &_transformed_b, _params);
- break;
-#ifdef __aarch64__
- case DataType::U8:
- case DataType::QASYMM8:
- if(use_dot)
- {
- _prepare_b = instantiate_prepareB<uint8_t, true>(_b, &_transformed_b, _params);
- }
- else
- {
- _prepare_b = instantiate_prepareB<uint8_t, false>(_b, &_transformed_b, _params);
- }
- break;
- case DataType::S8:
- if(use_dot)
- {
- _prepare_b = instantiate_prepareB<int8_t, true>(_b, &_transformed_b, _params);
- }
- else
- {
- _prepare_b = instantiate_prepareB<int8_t, false>(_b, &_transformed_b, _params);
- }
- break;
-#endif /* __aarch64__ */
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
- case DataType::F16:
- _prepare_b = instantiate_prepareB<__fp16>(_b, &_transformed_b, _params);
- break;
-#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
- default:
- ARM_COMPUTE_ERROR("DataType not supported");
- break;
- }
+
+ _prepare_b = strategy->instantiate_prepareB(b, &_transformed_b, _params, ci);
ARM_COMPUTE_ERROR_ON(_prepare_b == nullptr);
if(_pretranspose_b)
@@ -463,51 +405,11 @@
_memory_group.manage(&_transformed_a);
_memory_group.manage(&_tmp_c);
- switch(input_type)
- {
- case DataType::F32:
- _transform_a = instantiate_transformA<float>(_a, &_transformed_a, _block_walker, _params);
- _matrix_multiply = instantiate_matrix_multiply<float, float>(&_transformed_a, &_transformed_b, &_tmp_c, c, _block_walker, _block_sizes, _params, pretranspose_b, alpha, beta);
- break;
-#ifdef __aarch64__
- case DataType::U8:
- case DataType::QASYMM8:
- if(use_dot)
- {
- _transform_a = instantiate_transformA<uint8_t, true>(_a, &_transformed_a, _block_walker, _params);
- _matrix_multiply = instantiate_matrix_multiply<uint8_t, uint32_t, true>(&_transformed_a, &_transformed_b, &_tmp_c, c, _block_walker, _block_sizes, _params, pretranspose_b, alpha, beta);
- }
- else
- {
- _transform_a = instantiate_transformA<uint8_t, false>(_a, &_transformed_a, _block_walker, _params);
- _matrix_multiply = instantiate_matrix_multiply<uint8_t, uint32_t, false>(&_transformed_a, &_transformed_b, &_tmp_c, c, _block_walker, _block_sizes, _params, pretranspose_b, alpha, beta);
- }
- break;
- case DataType::S8:
- if(use_dot)
- {
- _transform_a = instantiate_transformA<int8_t, true>(_a, &_transformed_a, _block_walker, _params);
- _matrix_multiply = instantiate_matrix_multiply<int8_t, int32_t, true>(&_transformed_a, &_transformed_b, &_tmp_c, c, _block_walker, _block_sizes, _params, pretranspose_b, alpha, beta);
- }
- else
- {
- _transform_a = instantiate_transformA<int8_t, false>(_a, &_transformed_a, _block_walker, _params);
- _matrix_multiply = instantiate_matrix_multiply<int8_t, int32_t, false>(&_transformed_a, &_transformed_b, &_tmp_c, c, _block_walker, _block_sizes, _params, pretranspose_b, alpha, beta);
- }
- break;
-#endif /* __aarch64__ */
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
- case DataType::F16:
- _transform_a = instantiate_transformA<__fp16>(_a, &_transformed_a, _block_walker, _params);
- _matrix_multiply = instantiate_matrix_multiply<__fp16, __fp16>(&_transformed_a, &_transformed_b, &_tmp_c, c, _block_walker, _block_sizes, _params, pretranspose_b, alpha, beta);
- break;
- break;
-#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
- default:
- break;
- }
+ _transform_a = strategy->instantiate_transformA(_a, &_transformed_a, _block_walker, _params);
+ _matrix_multiply = strategy->instantiate_matrix_multiply(&_transformed_a, &_transformed_b, &_tmp_c, c, _block_walker, _block_sizes, _params, alpha, beta, pretranspose_b, num_threads);
ARM_COMPUTE_ERROR_ON(_transform_a == nullptr);
ARM_COMPUTE_ERROR_ON(_matrix_multiply == nullptr);
+
_transformed_a.allocator()->allocate();
_tmp_c.allocator()->allocate();
if(!_pretranspose_b)